diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml
index ccaff510c120a..cc2c2bd86416d 100644
--- a/.github/actions/docker-custom-build-and-push/action.yml
+++ b/.github/actions/docker-custom-build-and-push/action.yml
@@ -97,10 +97,11 @@ runs:
         cache-to: |
           type=inline
     - name: Upload image locally for testing (if not publishing)
-      uses: ishworkh/docker-image-artifact-upload@v1
+      uses: ishworkh/container-image-artifact-upload@v2.0.0
       if: ${{ inputs.publish != 'true' }}
       with:
         image: ${{ steps.single_tag.outputs.SINGLE_TAG }}
+        retention_days: "2"
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
index e75bafdac9628..26fcceb8aeab7 100644
--- a/.github/workflows/airflow-plugin.yml
+++ b/.github/workflows/airflow-plugin.yml
@@ -27,33 +27,27 @@ jobs:
   airflow-plugin:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
         include:
           # Note: this should be kept in sync with tox.ini.
           - python-version: "3.8"
-            extra_pip_requirements: "apache-airflow~=2.1.4"
-            extra_pip_extras: plugin-v1
-          - python-version: "3.8"
-            extra_pip_requirements: "apache-airflow~=2.2.4"
-            extra_pip_extras: plugin-v1
+            extra_pip_requirements: "apache-airflow~=2.3.4"
+            extra_pip_extras: test-airflow23
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.4.3"
-            extra_pip_extras: plugin-v2,test-airflow24
+            extra_pip_extras: test-airflow24
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.11"
             extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
-            extra_pip_extras: plugin-v2
+          - python-version: "3.11"
+            extra_pip_requirements: "apache-airflow~=2.10.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
       fail-fast: false
     steps:
       - name: Set up JDK 17
@@ -74,7 +68,7 @@ jobs:
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
         with:
           name: Test Results (Airflow Plugin ${{ matrix.python-version}})
@@ -85,10 +79,10 @@ jobs:
             !**/binary/**
       - name: Upload coverage to Codecov
         if: always()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: airflow,airflow-${{ matrix.extra_pip_extras }}
           name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
@@ -98,7 +92,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 52148ef1b91f9..98071b536a336 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -67,19 +67,25 @@ jobs:
           timezoneLinux: ${{ matrix.timezone }}
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
+      - name: Install dependencies
+        run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v5
-        if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
-        with:
-          python-version: "3.10"
-          cache: pip
       - name: Gradle build (and test) for NOT metadata ingestion
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        # datahub-schematron:cli excluded due to dependency on metadata-ingestion
         run: |
           ./gradlew build \
             -x :metadata-ingestion:build \
@@ -97,13 +103,14 @@ jobs:
             -x :metadata-ingestion-modules:gx-plugin:check \
             -x :datahub-frontend:build \
             -x :datahub-web-react:build \
+            -x :metadata-integration:java:datahub-schematron:cli:test \
             --parallel
       - name: Gradle build (and test) for frontend
         if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         run: |
           ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
         env:
-          NODE_OPTIONS: "--max-old-space-size=3072"
+          NODE_OPTIONS: "--max-old-space-size=4096"
       - name: Gradle compile (jdk8) for legacy Spark
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
@@ -119,6 +126,16 @@ jobs:
             !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: ./build/coverage-reports/
+          fail_ci_if_error: false
+          flags: ${{ matrix.timezone }}
+          name: ${{ matrix.command }}
+          verbose: true
 
   quickstart-compose-validation:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
index becf8126dc45b..dc770f7fc83a6 100644
--- a/.github/workflows/check-datahub-jars.yml
+++ b/.github/workflows/check-datahub-jars.yml
@@ -5,12 +5,12 @@ on:
     branches:
       - master
     paths:
-      - "metadata-integration"
+      - "metadata-integration/**"
   pull_request:
     branches:
       - "**"
     paths:
-      - "metadata-integration"
+      - "metadata-integration/**"
   release:
     types: [published]
 
@@ -28,16 +28,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
+      - name: Install dependencies
+        run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
       - name: check ${{ matrix.command }} jar
         run: |
           ./gradlew :metadata-integration:java:${{ matrix.command }}:build --info
+          ./gradlew :metadata-integration:java:${{ matrix.command }}:checkShadowJar
           ./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc
diff --git a/.github/workflows/contributor-open-pr-comment.yml b/.github/workflows/contributor-open-pr-comment.yml
new file mode 100644
index 0000000000000..2f700290ee0f2
--- /dev/null
+++ b/.github/workflows/contributor-open-pr-comment.yml
@@ -0,0 +1,42 @@
+name: PR Comment
+
+on:
+  pull_request:
+    types: [opened]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  post-pr-opened-comment:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Get and Format Username (PR only)
+        if: github.event_name == 'pull_request'
+        run: |
+          formatted_username=$(echo "${{ github.event.pull_request.user.login }}" | tr '[:upper:]' '[:lower:]' | sed 's/ /-/g')
+          echo "FORMATTED_USERNAME=$formatted_username" >> $GITHUB_ENV
+
+      - name: Create Comment (PR only)
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            if (context.payload.pull_request) {
+              const prUser = process.env.FORMATTED_USERNAME;
+              const url = `https://contributors.datahubproject.io/${prUser}`;
+              const body = `Hello @${prUser} :smile: \n\n Thank you so much for opening a pull request!\n\n![Image](https://contributors.datahubproject.io/api/og?userId=${{ github.event.pull_request.user.login }})\nYou can check out your contributor card and see all your past stats [here](${url})!`;
+                         
+              // Create a comment on the PR
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.payload.pull_request.number,
+                body: body
+              });
+            } else {
+              console.log('Not a pull request event.');
+            }
diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
index 6c7f197556bbe..ae9a0b1605cdf 100644
--- a/.github/workflows/dagster-plugin.yml
+++ b/.github/workflows/dagster-plugin.yml
@@ -27,13 +27,12 @@ jobs:
   dagster-plugin:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.9", "3.11"]
         include:
-          - python-version: "3.8"
+          - python-version: "3.9"
             extraPythonRequirement: "dagster>=1.3.3"
           - python-version: "3.11"
             extraPythonRequirement: "dagster>=1.3.3"
@@ -44,7 +43,8 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: actions/checkout@v4
+      - uses: gradle/actions/setup-gradle@v3
+      - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -56,8 +56,8 @@ jobs:
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
-      - uses: actions/upload-artifact@v3
-        if: ${{ always() && matrix.python-version == '3.11' }}
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
         with:
           name: Test Results (dagster Plugin ${{ matrix.python-version}})
           path: |
@@ -66,10 +66,10 @@ jobs:
             **/junit.*.xml
       - name: Upload coverage to Codecov
         if: always()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: dagster-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
           name: pytest-dagster
@@ -79,7 +79,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml
index 956f3f7b1c390..c028bfb55d48d 100644
--- a/.github/workflows/docker-postgres-setup.yml
+++ b/.github/workflows/docker-postgres-setup.yml
@@ -52,7 +52,7 @@ jobs:
         with:
           images: |
             acryldata/datahub-postgres-setup
-          tags: ${{ needs.setup.outputs.tag }}
+          image_tag: ${{ needs.setup.outputs.tag }}
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
           password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
           publish: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
index d1c16b567158a..03a9b3afc3bc5 100644
--- a/.github/workflows/docker-unified.yml
+++ b/.github/workflows/docker-unified.yml
@@ -144,6 +144,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
@@ -181,14 +186,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -208,6 +215,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
@@ -245,14 +257,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -272,6 +286,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
@@ -309,14 +328,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -336,6 +357,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
@@ -373,14 +399,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -400,6 +428,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true'}}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
@@ -413,7 +446,7 @@ jobs:
           ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
           mv ./datahub-frontend/build/distributions/datahub-frontend-*.zip datahub-frontend.zip
         env:
-          NODE_OPTIONS: "--max-old-space-size=3072"
+          NODE_OPTIONS: "--max-old-space-size=4096"
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -439,14 +472,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: actions/checkout@v4
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -466,6 +501,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Build and push
@@ -493,14 +533,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -520,6 +562,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Build and push
@@ -547,14 +594,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -574,6 +623,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' ) }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Build and push
@@ -601,14 +655,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -630,6 +686,11 @@ jobs:
     needs: setup
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Build and push Base Image
@@ -658,10 +719,15 @@ jobs:
     needs: [setup, datahub_ingestion_base_build]
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Download Base Image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}
@@ -701,10 +767,15 @@ jobs:
     needs: [setup, datahub_ingestion_base_build]
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Download Base Image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}
@@ -744,19 +815,28 @@ jobs:
     needs: [setup, datahub_ingestion_base_slim_build]
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
+      - name: Check out the repo
+        uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - name: Check out the repo
-        uses: acryldata/sane-checkout-action@v3
       - name: Build codegen
         if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish =='true' }}
         run: ./gradlew :metadata-ingestion:codegen
       - name: Download Base Image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
@@ -803,14 +883,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image Slim Image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
       - name: Run Trivy vulnerability scanner Slim Image
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
           format: "template"
@@ -834,19 +916,28 @@ jobs:
     needs: [setup, datahub_ingestion_base_full_build]
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
+      - name: Check out the repo
+        uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - name: Check out the repo
-        uses: acryldata/sane-checkout-action@v3
       - name: Build codegen
         if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
         run: ./gradlew :metadata-ingestion:codegen
       - name: Download Base Image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}
@@ -891,14 +982,16 @@ jobs:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: acryldata/sane-checkout-action@v3
       - name: Download image Full Image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.datahub_ingestion_full_build.outputs.needs_artifact_download == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
       - name: Run Trivy vulnerability scanner Full Image
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
           format: "template"
@@ -963,16 +1056,16 @@ jobs:
         run: df -h . && docker images
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-          cache: "pip"
       - name: Login to DockerHub
         uses: docker/login-action@v3
         if: ${{ needs.setup.outputs.docker-login == 'true' }}
@@ -986,47 +1079,47 @@ jobs:
       - name: Disk Check
         run: df -h . && docker images
       - name: Download GMS image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) && needs.gms_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download Frontend image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) && needs.frontend_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download Kafka Setup image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) && needs.kafka_setup_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download Mysql Setup image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) && needs.mysql_setup_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download Elastic Setup image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) && needs.elasticsearch_setup_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download MCE Consumer image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) && needs.mce_consumer_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download MAE Consumer image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) && needs.mae_consumer_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download upgrade image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) && needs.datahub_upgrade_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download datahub-ingestion-slim image
-        uses: ishworkh/docker-image-artifact-download@v1
+        uses: ishworkh/container-image-artifact-download@v2.0.0
         if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' && needs.datahub_ingestion_slim_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
index dbeff7a9c0f9f..e6044badb1b41 100644
--- a/.github/workflows/documentation.yml
+++ b/.github/workflows/documentation.yml
@@ -44,7 +44,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-          cache: pip
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - name: Install Python dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Build Docs
diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml
index 99121f81099f2..2fd814a076485 100644
--- a/.github/workflows/gx-plugin.yml
+++ b/.github/workflows/gx-plugin.yml
@@ -27,7 +27,6 @@ jobs:
   gx-plugin:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
@@ -39,6 +38,8 @@ jobs:
             extraPythonRequirement: "great-expectations~=0.16.0 numpy~=1.26.0"
           - python-version: "3.11"
             extraPythonRequirement: "great-expectations~=0.17.0"
+          - python-version: "3.11"
+            extraPythonRequirement: "great-expectations~=0.18.0"
       fail-fast: false
     steps:
       - name: Set up JDK 17
@@ -46,7 +47,8 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: actions/checkout@v4
+      - uses: gradle/actions/setup-gradle@v3
+      - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -58,7 +60,7 @@ jobs:
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }}
         with:
           name: Test Results (GX Plugin ${{ matrix.python-version}})
@@ -68,10 +70,10 @@ jobs:
             **/junit.*.xml
       - name: Upload coverage to Codecov
         if: always()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: gx-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
           name: pytest-gx
@@ -81,7 +83,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
index f0d7fd2b51184..106cba1473982 100644
--- a/.github/workflows/metadata-ingestion.yml
+++ b/.github/workflows/metadata-ingestion.yml
@@ -26,8 +26,8 @@ concurrency:
 jobs:
   metadata-ingestion:
     runs-on: ubuntu-latest
+    timeout-minutes: 40
     env:
-      SPARK_VERSION: 3.5.3
       DATAHUB_TELEMETRY_ENABLED: false
       # TODO: Enable this once the test is fixed.
       # DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
@@ -61,16 +61,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
       - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/uv
           key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install package
@@ -88,9 +83,9 @@ jobs:
           df -hl
           docker image ls
           docker system df
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
-          name: Test Results (metadata ingestion ${{ matrix.python-version }})
+          name: Test Results (metadata ingestion ${{ matrix.python-version }} ${{ matrix.command }})
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**
@@ -98,20 +93,20 @@ jobs:
             !**/binary/**
       - name: Upload coverage to Codecov
         if: ${{ always() && matrix.python-version == '3.10' }}
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: pytest-${{ matrix.command }}
-          name: pytest-${{ matrix.command }}
+          name: pytest-${{ matrix.python-version }}-${{ matrix.command }}
           verbose: true
 
   event-file:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
index 7018b42949e89..2225baecde64c 100644
--- a/.github/workflows/metadata-io.yml
+++ b/.github/workflows/metadata-io.yml
@@ -57,17 +57,16 @@ jobs:
       - name: Disk Check
         run: df -h . && docker images
       - uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v5
-        if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
-        with:
-          python-version: "3.10"
-          cache: "pip"
       - name: Gradle build (and test)
         run: |
           ./gradlew :metadata-io:test
@@ -82,6 +81,15 @@ jobs:
             !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
+      - name: Upload coverage to Codecov
+        if: ${{ always()}}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: ./build/coverage-reports/
+          fail_ci_if_error: false
+          name: metadata-io-test
+          verbose: true
 
   event-file:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
index 1ae3edae7aa90..de7ad21b3e67b 100644
--- a/.github/workflows/pr-labeler.yml
+++ b/.github/workflows/pr-labeler.yml
@@ -29,7 +29,6 @@ jobs:
                 "swaroopjagadish",
                 "treff7es",
                 "yoonhyejin",
-                "eboneil",
                 "gabe-lyons",
                 "hsheth2",
                 "jjoyce0510",
@@ -37,15 +36,17 @@ jobs:
                 "pedro93",
                 "RyanHolstien",
                 "sakethvarma397",
-                "Kunal-kankriya",
                 "purnimagarg1",
-                "dushayntAW",
                 "sagar-salvi-apptware",
                 "kushagra-apptware",
                 "Salman-Apptware",
                 "mayurinehate",
                 "noggi",
-                "skrydal"
+                "skrydal",
+                "kevinkarchacryl",
+                "sgomezvillamor",
+                "acrylJonny",
+                "chakru-r"
               ]'), 
               github.actor
             ) 
diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml
index 9ecde0887142d..d77142a1f00de 100644
--- a/.github/workflows/prefect-plugin.yml
+++ b/.github/workflows/prefect-plugin.yml
@@ -27,24 +27,20 @@ jobs:
   prefect-plugin:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
-        python-version: ["3.8", "3.11"]
-        include:
-          - python-version: "3.8"
-          - python-version: "3.11"
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
       fail-fast: false
     steps:
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -55,7 +51,7 @@ jobs:
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ always() && matrix.python-version == '3.11'}}
         with:
           name: Test Results (Prefect Plugin ${{ matrix.python-version}})
@@ -66,12 +62,12 @@ jobs:
             !**/binary/**
       - name: Upload coverage to Codecov
         if: always()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
-          flags: prefect,prefect-${{ matrix.extra_pip_extras }}
+          flags: prefect,prefect-${{ matrix.python-version }}
           name: pytest-prefect-${{ matrix.python-version }}
           verbose: true
 
@@ -79,7 +75,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
index eb57c29e151ae..393f9d993e2a2 100644
--- a/.github/workflows/publish-datahub-jars.yml
+++ b/.github/workflows/publish-datahub-jars.yml
@@ -196,3 +196,52 @@ jobs:
           echo signingKey=$SIGNING_KEY >> gradle.properties
           ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:custom-plugin-lib:publish
           ./gradlew :metadata-integration:java:custom-plugin-lib:closeAndReleaseRepository --info
+  publish-java8:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    needs: ["check-secret", "setup", "publish"]
+    if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
+    steps:
+      - uses: acryldata/sane-checkout-action@v3
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - uses: gradle/actions/setup-gradle@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - name: checkout upstream repo
+        run: |
+          git remote add upstream https://github.com/datahub-project/datahub.git
+          git fetch upstream --tags --force --filter=tree:0
+      - name: publish datahub-client jar snapshot
+        if: ${{ github.event_name != 'release' }}
+        env:
+          RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
+          RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
+          SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          echo signingKey=$SIGNING_KEY >> gradle.properties
+          ./gradlew :metadata-integration:java:datahub-client:printVersion -PjavaClassVersionDefault=8 -ParchiveAppendix=java8
+          ./gradlew :metadata-integration:java:datahub-client:publish -PjavaClassVersionDefault=8 -ParchiveAppendix=java8
+      - name: release datahub-client jar
+        if: ${{ github.event_name == 'release' }}
+        env:
+          RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
+          RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
+          SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          echo signingKey=$SIGNING_KEY >> gradle.properties
+          ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-client:publish -PjavaClassVersionDefault=8 -ParchiveAppendix=java8
+          ./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info -PjavaClassVersionDefault=8 -ParchiveAppendix=java8
\ No newline at end of file
diff --git a/.github/workflows/python-build-pages.yml b/.github/workflows/python-build-pages.yml
new file mode 100644
index 0000000000000..8971722c374fb
--- /dev/null
+++ b/.github/workflows/python-build-pages.yml
@@ -0,0 +1,64 @@
+name: Python Build
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/python-build-pages.yml"
+      - "metadata-ingestion/**"
+      - "metadata-ingestion-modules/**"
+      - "metadata-models/**"
+  pull_request:
+    branches:
+      - "**"
+    paths:
+      - ".github/workflows/python-build-pages.yml"
+      - "metadata-ingestion/**"
+      - "metadata-ingestion-modules/**"
+      - "metadata-models/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deploy-pages:
+    runs-on: ubuntu-latest
+    if: ${{ vars.CLOUDFLARE_WHEELS_PROJECT_NAME != '' }}
+
+    name: Python Wheels
+    permissions:
+      contents: read
+      pull-requests: read
+      deployments: write
+    steps:
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - uses: gradle/actions/setup-gradle@v3
+      - uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
+      - name: Build Python wheel site
+        run: |
+          ./gradlew :python-build:buildSite
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Publish
+        uses: cloudflare/pages-action@v1
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          projectName: ${{ vars.CLOUDFLARE_WHEELS_PROJECT_NAME }}
+          workingDirectory: python-build
+          directory: site
+          gitHubToken: ${{ github.token }}
diff --git a/build.gradle b/build.gradle
index d7fbbb6682e04..a3d807a733349 100644
--- a/build.gradle
+++ b/build.gradle
@@ -34,6 +34,7 @@ buildscript {
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
   ext.pegasusVersion = '29.57.0'
   ext.mavenVersion = '3.6.3'
+  ext.versionGradle = '8.11.1'
   ext.springVersion = '6.1.13'
   ext.springBootVersion = '3.2.9'
   ext.springKafkaVersion = '3.1.6'
@@ -45,7 +46,10 @@ buildscript {
   ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
   ext.jacksonVersion = '2.15.3'
   ext.jettyVersion = '11.0.21'
+  // see also datahub-frontend/play.gradle
   ext.playVersion = '2.8.22'
+  ext.playScalaVersion = '2.13'
+  ext.akkaVersion = '2.6.21' // 2.7.0+ has incompatible license
   ext.log4jVersion = '2.23.1'
   ext.slf4jVersion = '1.7.36'
   ext.logbackClassic = '1.4.14'
@@ -54,7 +58,7 @@ buildscript {
   ext.hazelcastVersion = '5.3.6'
   ext.ebeanVersion = '15.5.2'
   ext.googleJavaFormatVersion = '1.18.1'
-  ext.openLineageVersion = '1.19.0'
+  ext.openLineageVersion = '1.25.0'
   ext.logbackClassicJava8 = '1.2.12'
 
   ext.docker_registry = 'acryldata'
@@ -75,7 +79,7 @@ buildscript {
 
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.1'
-  id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
+  id 'com.gradleup.shadow' version '8.3.5' apply false
   id 'com.palantir.docker' version '0.35.0' apply false
   id 'com.avast.gradle.docker-compose' version '0.17.6'
   id "com.diffplug.spotless" version "6.23.3"
@@ -103,12 +107,19 @@ project.ext.spec = [
 ]
 
 project.ext.externalDependency = [
-    'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10',
+    'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", // max version due to licensing
+    'akkaActor': "com.typesafe.akka:akka-actor_$playScalaVersion:$akkaVersion",
+    'akkaStream': "com.typesafe.akka:akka-stream_$playScalaVersion:$akkaVersion",
+    'akkaActorTyped': "com.typesafe.akka:akka-actor-typed_$playScalaVersion:$akkaVersion",
+    'akkaSlf4j': "com.typesafe.akka:akka-slf4j_$playScalaVersion:$akkaVersion",
+    'akkaJackson': "com.typesafe.akka:akka-serialization-jackson_$playScalaVersion:$akkaVersion",
+    'akkaParsing': "com.typesafe.akka:akka-parsing_$playScalaVersion:$akkaVersion",
+    'akkaProtobuf': "com.typesafe.akka:akka-protobuf-v3_$playScalaVersion:$akkaVersion",
     'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
     'antlr4': 'org.antlr:antlr4:4.9.3',
     'assertJ': 'org.assertj:assertj-core:3.11.1',
-    'avro': 'org.apache.avro:avro:1.11.3',
-    'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
+    'avro': 'org.apache.avro:avro:1.11.4',
+    'avroCompiler': 'org.apache.avro:avro-compiler:1.11.4',
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.0.3',
     'awsS3': 'software.amazon.awssdk:s3:2.26.21',
@@ -212,18 +223,18 @@ project.ext.externalDependency = [
     'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
     'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
     'picocli': 'info.picocli:picocli:4.5.0',
-    'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
-    'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_2.12:$playVersion",
-    'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
-    'playDocs': "com.typesafe.play:play-docs_2.12:$playVersion",
-    'playGuice': "com.typesafe.play:play-guice_2.12:$playVersion",
-    'playJavaJdbc': "com.typesafe.play:play-java-jdbc_2.12:$playVersion",
-    'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_2.12:$playVersion",
-    'playServer': "com.typesafe.play:play-server_2.12:$playVersion",
-    'playTest': "com.typesafe.play:play-test_2.12:$playVersion",
-    'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion",
-    'pac4j': 'org.pac4j:pac4j-oidc:4.5.8',
-    'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
+    'playCache': "com.typesafe.play:play-cache_$playScalaVersion:$playVersion",
+    'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_$playScalaVersion:$playVersion",
+    'playWs': "com.typesafe.play:play-ahc-ws-standalone_$playScalaVersion:2.1.10",
+    'playDocs': "com.typesafe.play:play-docs_$playScalaVersion:$playVersion",
+    'playGuice': "com.typesafe.play:play-guice_$playScalaVersion:$playVersion",
+    'playJavaJdbc': "com.typesafe.play:play-java-jdbc_$playScalaVersion:$playVersion",
+    'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_$playScalaVersion:$playVersion",
+    'playServer': "com.typesafe.play:play-server_$playScalaVersion:$playVersion",
+    'playTest': "com.typesafe.play:play-test_$playScalaVersion:$playVersion",
+    'playFilters': "com.typesafe.play:filters-helpers_$playScalaVersion:$playVersion",
+    'pac4j': 'org.pac4j:pac4j-oidc:6.0.6',
+    'playPac4j': "org.pac4j:play-pac4j_$playScalaVersion:12.0.0-PLAY2.8",
     'postgresql': 'org.postgresql:postgresql:42.3.9',
     'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
     'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
@@ -348,6 +359,7 @@ allprojects {
       }
     }
   }
+
 }
 
 configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
@@ -361,6 +373,7 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
     exclude group: "org.slf4j", module: "slf4j-log4j12"
     exclude group: "org.slf4j", module: "slf4j-nop"
     exclude group: "org.slf4j", module: "slf4j-ext"
+    exclude group: "org.codehaus.jackson", module: "jackson-mapper-asl"
 
     resolutionStrategy.force externalDependency.antlr4Runtime
     resolutionStrategy.force externalDependency.antlr4
@@ -391,13 +404,14 @@ subprojects {
       implementation externalDependency.annotationApi
       constraints {
         implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
-        implementation('io.netty:netty-all:4.1.114.Final')
+        implementation('io.netty:netty-all:4.1.115.Final')
         implementation('org.apache.commons:commons-compress:1.27.1')
         implementation('org.apache.velocity:velocity-engine-core:2.4')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')
         implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
         implementation("com.fasterxml.jackson.core:jackson-dataformat-cbor:$jacksonVersion")
         implementation(externalDependency.commonsIo)
+        implementation(externalDependency.protobuf)
       }
     }
 
@@ -406,6 +420,8 @@ subprojects {
         googleJavaFormat()
         target project.fileTree(project.projectDir) {
           include 'src/**/*.java'
+          include 'app/**/*.java'
+          include 'test/**/*.java'
           exclude 'src/**/resources/'
           exclude 'src/**/generated/'
           exclude 'src/**/mainGeneratedDataTemplate/'
@@ -485,3 +501,8 @@ subprojects {
     }
   }
 }
+
+wrapper {
+  gradleVersion = project.versionGradle
+  distributionType = Wrapper.DistributionType.ALL
+}
diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java
index d0d17fda26392..b95515684f01f 100644
--- a/datahub-frontend/app/auth/AuthModule.java
+++ b/datahub-frontend/app/auth/AuthModule.java
@@ -13,6 +13,7 @@
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
 import com.google.inject.name.Named;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.entity.client.SystemRestliEntityClient;
 import com.linkedin.metadata.models.registry.EmptyEntityRegistry;
@@ -21,26 +22,29 @@
 import com.linkedin.util.Configuration;
 import config.ConfigurationProvider;
 import controllers.SsoCallbackController;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-
 import io.datahubproject.metadata.context.ActorContext;
 import io.datahubproject.metadata.context.AuthorizationContext;
 import io.datahubproject.metadata.context.EntityRegistryContext;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.OperationContextConfig;
+import io.datahubproject.metadata.context.RetrieverContext;
 import io.datahubproject.metadata.context.SearchContext;
+import io.datahubproject.metadata.context.ValidationContext;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.pac4j.core.config.Config;
 import org.pac4j.core.context.session.SessionStore;
+import org.pac4j.core.profile.ProfileManager;
+import org.pac4j.core.util.serializer.JavaSerializer;
 import org.pac4j.play.LogoutController;
 import org.pac4j.play.http.PlayHttpActionAdapter;
 import org.pac4j.play.store.PlayCacheSessionStore;
 import org.pac4j.play.store.PlayCookieSessionStore;
-import org.pac4j.play.store.PlaySessionStore;
 import org.pac4j.play.store.ShiroAesDataEncrypter;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
 import play.Environment;
@@ -62,14 +66,16 @@ public class AuthModule extends AbstractModule {
   private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider";
   private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval";
   private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries";
-  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize";
-  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY = "entityClient.restli.get.batchConcurrency";
+  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE =
+      "entityClient.restli.get.batchSize";
+  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY =
+      "entityClient.restli.get.batchConcurrency";
   private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings";
 
-  private final com.typesafe.config.Config _configs;
+  private final com.typesafe.config.Config configs;
 
   public AuthModule(final Environment environment, final com.typesafe.config.Config configs) {
-    _configs = configs;
+    this.configs = configs;
   }
 
   @Override
@@ -83,13 +89,13 @@ protected void configure() {
      * the response will be rejected by the browser. Default to PlayCacheCookieStore so that
      * datahub-frontend container remains as a stateless service
      */
-    String sessionStoreProvider = _configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF);
+    String sessionStoreProvider = configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF);
 
     if (sessionStoreProvider.equals("PlayCacheSessionStore")) {
       final PlayCacheSessionStore playCacheSessionStore =
           new PlayCacheSessionStore(getProvider(SyncCacheApi.class));
       bind(SessionStore.class).toInstance(playCacheSessionStore);
-      bind(PlaySessionStore.class).toInstance(playCacheSessionStore);
+      bind(PlayCacheSessionStore.class).toInstance(playCacheSessionStore);
     } else {
       PlayCookieSessionStore playCacheCookieStore;
       try {
@@ -97,17 +103,18 @@ protected void configure() {
         // hash the input to generate a fixed-length string. Then, we convert
         // it to hex and slice the first 16 bytes, because AES key length must strictly
         // have a specific length.
-        final String aesKeyBase = _configs.getString(PAC4J_AES_KEY_BASE_CONF);
+        final String aesKeyBase = configs.getString(PAC4J_AES_KEY_BASE_CONF);
         final String aesKeyHash =
             DigestUtils.sha256Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8));
         final String aesEncryptionKey = aesKeyHash.substring(0, 16);
         playCacheCookieStore =
             new PlayCookieSessionStore(new ShiroAesDataEncrypter(aesEncryptionKey.getBytes()));
+        playCacheCookieStore.setSerializer(new JavaSerializer());
       } catch (Exception e) {
         throw new RuntimeException("Failed to instantiate Pac4j cookie session store!", e);
       }
       bind(SessionStore.class).toInstance(playCacheCookieStore);
-      bind(PlaySessionStore.class).toInstance(playCacheCookieStore);
+      bind(PlayCookieSessionStore.class).toInstance(playCacheCookieStore);
     }
 
     try {
@@ -132,9 +139,12 @@ protected void configure() {
 
   @Provides
   @Singleton
-  protected Config provideConfig() {
+  protected Config provideConfig(@Nonnull SessionStore sessionStore) {
     Config config = new Config();
+    config.setSessionStoreFactory(parameters -> sessionStore);
     config.setHttpActionAdapter(new PlayHttpActionAdapter());
+    config.setProfileManagerFactory(ProfileManager::new);
+
     return config;
   }
 
@@ -144,7 +154,7 @@ protected SsoManager provideSsoManager(
       Authentication systemAuthentication, CloseableHttpClient httpClient) {
     SsoManager manager =
         new SsoManager(
-            _configs, systemAuthentication, getSsoSettingsRequestUrl(_configs), httpClient);
+            configs, systemAuthentication, getSsoSettingsRequestUrl(configs), httpClient);
     manager.initializeSsoProvider();
     return manager;
   }
@@ -154,8 +164,8 @@ protected SsoManager provideSsoManager(
   protected Authentication provideSystemAuthentication() {
     // Returns an instance of Authentication used to authenticate system initiated calls to Metadata
     // Service.
-    String systemClientId = _configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH);
-    String systemSecret = _configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH);
+    String systemClientId = configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH);
+    String systemSecret = configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH);
     final Actor systemActor =
         new Actor(ActorType.USER, systemClientId); // TODO: Change to service actor once supported.
     return new Authentication(
@@ -168,26 +178,26 @@ protected Authentication provideSystemAuthentication() {
   @Singleton
   @Named("systemOperationContext")
   protected OperationContext provideOperationContext(
-          final Authentication systemAuthentication,
-          final ConfigurationProvider configurationProvider) {
+      final Authentication systemAuthentication,
+      final ConfigurationProvider configurationProvider) {
     ActorContext systemActorContext =
-            ActorContext.builder()
-                    .systemAuth(true)
-                    .authentication(systemAuthentication)
-                    .build();
-    OperationContextConfig systemConfig = OperationContextConfig.builder()
+        ActorContext.builder().systemAuth(true).authentication(systemAuthentication).build();
+    OperationContextConfig systemConfig =
+        OperationContextConfig.builder()
             .viewAuthorizationConfiguration(configurationProvider.getAuthorization().getView())
             .allowSystemAuthentication(true)
             .build();
 
     return OperationContext.builder()
-            .operationContextConfig(systemConfig)
-            .systemActorContext(systemActorContext)
-            // Authorizer.EMPTY is fine since it doesn't actually apply to system auth
-            .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
-            .searchContext(SearchContext.EMPTY)
-            .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
-            .build(systemAuthentication);
+        .operationContextConfig(systemConfig)
+        .systemActorContext(systemActorContext)
+        // Authorizer.EMPTY is fine since it doesn't actually apply to system auth
+        .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
+        .searchContext(SearchContext.EMPTY)
+        .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
+        .validationContext(ValidationContext.builder().alternateValidation(false).build())
+        .retrieverContext(RetrieverContext.EMPTY)
+        .build(systemAuthentication);
   }
 
   @Provides
@@ -206,11 +216,13 @@ protected SystemEntityClient provideEntityClient(
 
     return new SystemRestliEntityClient(
         buildRestliClient(),
-        new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
-        _configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
-        configurationProvider.getCache().getClient().getEntityClient(),
-        Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)),
-        Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY)));
+        EntityClientConfig.builder()
+            .backoffPolicy(new ExponentialBackoff(configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)))
+            .retryCount(configs.getInt(ENTITY_CLIENT_NUM_RETRIES))
+            .batchGetV2Size(configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE))
+            .batchGetV2Concurrency(2)
+            .build(),
+        configurationProvider.getCache().getClient().getEntityClient());
   }
 
   @Provides
@@ -218,11 +230,11 @@ protected SystemEntityClient provideEntityClient(
   protected AuthServiceClient provideAuthClient(
       Authentication systemAuthentication, CloseableHttpClient httpClient) {
     // Init a GMS auth client
-    final String metadataServiceHost = getMetadataServiceHost(_configs);
+    final String metadataServiceHost = getMetadataServiceHost(configs);
 
-    final int metadataServicePort = getMetadataServicePort(_configs);
+    final int metadataServicePort = getMetadataServicePort(configs);
 
-    final boolean metadataServiceUseSsl = doesMetadataServiceUseSsl(_configs);
+    final boolean metadataServiceUseSsl = doesMetadataServiceUseSsl(configs);
 
     return new AuthServiceClient(
         metadataServiceHost,
@@ -241,22 +253,22 @@ protected CloseableHttpClient provideHttpClient() {
   private com.linkedin.restli.client.Client buildRestliClient() {
     final String metadataServiceHost =
         utils.ConfigUtil.getString(
-            _configs,
+            configs,
             METADATA_SERVICE_HOST_CONFIG_PATH,
             utils.ConfigUtil.DEFAULT_METADATA_SERVICE_HOST);
     final int metadataServicePort =
         utils.ConfigUtil.getInt(
-            _configs,
+            configs,
             utils.ConfigUtil.METADATA_SERVICE_PORT_CONFIG_PATH,
             utils.ConfigUtil.DEFAULT_METADATA_SERVICE_PORT);
     final boolean metadataServiceUseSsl =
         utils.ConfigUtil.getBoolean(
-            _configs,
+            configs,
             utils.ConfigUtil.METADATA_SERVICE_USE_SSL_CONFIG_PATH,
             ConfigUtil.DEFAULT_METADATA_SERVICE_USE_SSL);
     final String metadataServiceSslProtocol =
         utils.ConfigUtil.getString(
-            _configs,
+            configs,
             utils.ConfigUtil.METADATA_SERVICE_SSL_PROTOCOL_CONFIG_PATH,
             ConfigUtil.DEFAULT_METADATA_SERVICE_SSL_PROTOCOL);
     return DefaultRestliClientFactory.getRestLiClient(
diff --git a/datahub-frontend/app/auth/AuthUtils.java b/datahub-frontend/app/auth/AuthUtils.java
index 51bb784c61b3b..490f52bece651 100644
--- a/datahub-frontend/app/auth/AuthUtils.java
+++ b/datahub-frontend/app/auth/AuthUtils.java
@@ -75,6 +75,7 @@ public class AuthUtils {
   public static final String RESPONSE_MODE = "responseMode";
   public static final String USE_NONCE = "useNonce";
   public static final String READ_TIMEOUT = "readTimeout";
+  public static final String CONNECT_TIMEOUT = "connectTimeout";
   public static final String EXTRACT_JWT_ACCESS_TOKEN_CLAIMS = "extractJwtAccessTokenClaims";
   // Retained for backwards compatibility
   public static final String PREFERRED_JWS_ALGORITHM = "preferredJwsAlgorithm";
diff --git a/datahub-frontend/app/auth/CookieConfigs.java b/datahub-frontend/app/auth/CookieConfigs.java
index 63b2ce61aaf9b..e77e200144835 100644
--- a/datahub-frontend/app/auth/CookieConfigs.java
+++ b/datahub-frontend/app/auth/CookieConfigs.java
@@ -10,34 +10,34 @@ public class CookieConfigs {
   public static final String AUTH_COOKIE_SECURE = "play.http.session.secure";
   public static final boolean DEFAULT_AUTH_COOKIE_SECURE = false;
 
-  private final int _ttlInHours;
-  private final String _authCookieSameSite;
-  private final boolean _authCookieSecure;
+  private final int ttlInHours;
+  private final String authCookieSameSite;
+  private final boolean authCookieSecure;
 
   public CookieConfigs(final Config configs) {
-    _ttlInHours =
+    ttlInHours =
         configs.hasPath(SESSION_TTL_CONFIG_PATH)
             ? configs.getInt(SESSION_TTL_CONFIG_PATH)
             : DEFAULT_SESSION_TTL_HOURS;
-    _authCookieSameSite =
+    authCookieSameSite =
         configs.hasPath(AUTH_COOKIE_SAME_SITE)
             ? configs.getString(AUTH_COOKIE_SAME_SITE)
             : DEFAULT_AUTH_COOKIE_SAME_SITE;
-    _authCookieSecure =
+    authCookieSecure =
         configs.hasPath(AUTH_COOKIE_SECURE)
             ? configs.getBoolean(AUTH_COOKIE_SECURE)
             : DEFAULT_AUTH_COOKIE_SECURE;
   }
 
   public int getTtlInHours() {
-    return _ttlInHours;
+    return ttlInHours;
   }
 
   public String getAuthCookieSameSite() {
-    return _authCookieSameSite;
+    return authCookieSameSite;
   }
 
   public boolean getAuthCookieSecure() {
-    return _authCookieSecure;
+    return authCookieSecure;
   }
 }
diff --git a/datahub-frontend/app/auth/JAASConfigs.java b/datahub-frontend/app/auth/JAASConfigs.java
index 529bf98e1fdcf..dee4ded68808a 100644
--- a/datahub-frontend/app/auth/JAASConfigs.java
+++ b/datahub-frontend/app/auth/JAASConfigs.java
@@ -8,16 +8,16 @@ public class JAASConfigs {
 
   public static final String JAAS_ENABLED_CONFIG_PATH = "auth.jaas.enabled";
 
-  private Boolean _isEnabled = true;
+  private Boolean isEnabled = true;
 
   public JAASConfigs(final com.typesafe.config.Config configs) {
     if (configs.hasPath(JAAS_ENABLED_CONFIG_PATH)
         && !configs.getBoolean(JAAS_ENABLED_CONFIG_PATH)) {
-      _isEnabled = false;
+      isEnabled = false;
     }
   }
 
   public boolean isJAASEnabled() {
-    return _isEnabled;
+    return isEnabled;
   }
 }
diff --git a/datahub-frontend/app/auth/NativeAuthenticationConfigs.java b/datahub-frontend/app/auth/NativeAuthenticationConfigs.java
index 772c2c8f92f28..a7b8a8bc80067 100644
--- a/datahub-frontend/app/auth/NativeAuthenticationConfigs.java
+++ b/datahub-frontend/app/auth/NativeAuthenticationConfigs.java
@@ -7,17 +7,17 @@ public class NativeAuthenticationConfigs {
   public static final String NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH =
       "auth.native.signUp.enforceValidEmail";
 
-  private Boolean _isEnabled = true;
-  private Boolean _isEnforceValidEmailEnabled = true;
+  private Boolean isEnabled = true;
+  private Boolean isEnforceValidEmailEnabled = true;
 
   public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) {
     if (configs.hasPath(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH)) {
-      _isEnabled =
+      isEnabled =
           Boolean.parseBoolean(
               configs.getValue(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH).toString());
     }
     if (configs.hasPath(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH)) {
-      _isEnforceValidEmailEnabled =
+      isEnforceValidEmailEnabled =
           Boolean.parseBoolean(
               configs
                   .getValue(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH)
@@ -26,10 +26,10 @@ public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) {
   }
 
   public boolean isNativeAuthenticationEnabled() {
-    return _isEnabled;
+    return isEnabled;
   }
 
   public boolean isEnforceValidEmailEnabled() {
-    return _isEnforceValidEmailEnabled;
+    return isEnforceValidEmailEnabled;
   }
 }
diff --git a/datahub-frontend/app/auth/sso/SsoConfigs.java b/datahub-frontend/app/auth/sso/SsoConfigs.java
index 976d0826f2277..46a2b7bfd27e8 100644
--- a/datahub-frontend/app/auth/sso/SsoConfigs.java
+++ b/datahub-frontend/app/auth/sso/SsoConfigs.java
@@ -1,10 +1,9 @@
 package auth.sso;
 
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
 import static auth.AuthUtils.*;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 
 /**
  * Class responsible for extracting and validating top-level SSO related configurations. TODO:
@@ -25,72 +24,72 @@ public class SsoConfigs {
 
   private static final String DEFAULT_SUCCESS_REDIRECT_PATH = "/";
 
-  private final String _authBaseUrl;
-  private final String _authBaseCallbackPath;
-  private final String _authSuccessRedirectPath;
-  private final Boolean _oidcEnabled;
+  private final String authBaseUrl;
+  private final String authBaseCallbackPath;
+  private final String authSuccessRedirectPath;
+  private final Boolean oidcEnabled;
 
   public SsoConfigs(Builder<?> builder) {
-    _authBaseUrl = builder._authBaseUrl;
-    _authBaseCallbackPath = builder._authBaseCallbackPath;
-    _authSuccessRedirectPath = builder._authSuccessRedirectPath;
-    _oidcEnabled = builder._oidcEnabled;
+    authBaseUrl = builder.authBaseUrl;
+    authBaseCallbackPath = builder.authBaseCallbackPath;
+    authSuccessRedirectPath = builder.authSuccessRedirectPath;
+    oidcEnabled = builder.oidcEnabled;
   }
 
   public String getAuthBaseUrl() {
-    return _authBaseUrl;
+    return authBaseUrl;
   }
 
   public String getAuthBaseCallbackPath() {
-    return _authBaseCallbackPath;
+    return authBaseCallbackPath;
   }
 
   public String getAuthSuccessRedirectPath() {
-    return _authSuccessRedirectPath;
+    return authSuccessRedirectPath;
   }
 
   public Boolean isOidcEnabled() {
-    return _oidcEnabled;
+    return oidcEnabled;
   }
 
   public static class Builder<T extends Builder<T>> {
-    protected String _authBaseUrl = null;
-    private String _authBaseCallbackPath = DEFAULT_BASE_CALLBACK_PATH;
-    private String _authSuccessRedirectPath = DEFAULT_SUCCESS_REDIRECT_PATH;
-    protected Boolean _oidcEnabled = false;
-    private final ObjectMapper _objectMapper = new ObjectMapper();
+    protected String authBaseUrl = null;
+    private String authBaseCallbackPath = DEFAULT_BASE_CALLBACK_PATH;
+    private String authSuccessRedirectPath = DEFAULT_SUCCESS_REDIRECT_PATH;
+    protected Boolean oidcEnabled = false;
+    private final ObjectMapper objectMapper = new ObjectMapper();
     protected JsonNode jsonNode = null;
 
     // No need to check if changes are made since this method is only called at start-up.
     public Builder from(final com.typesafe.config.Config configs) {
       if (configs.hasPath(AUTH_BASE_URL_CONFIG_PATH)) {
-        _authBaseUrl = configs.getString(AUTH_BASE_URL_CONFIG_PATH);
+        authBaseUrl = configs.getString(AUTH_BASE_URL_CONFIG_PATH);
       }
       if (configs.hasPath(AUTH_BASE_CALLBACK_PATH_CONFIG_PATH)) {
-        _authBaseCallbackPath = configs.getString(AUTH_BASE_CALLBACK_PATH_CONFIG_PATH);
+        authBaseCallbackPath = configs.getString(AUTH_BASE_CALLBACK_PATH_CONFIG_PATH);
       }
       if (configs.hasPath(OIDC_ENABLED_CONFIG_PATH)) {
-        _oidcEnabled =
+        oidcEnabled =
             Boolean.TRUE.equals(Boolean.parseBoolean(configs.getString(OIDC_ENABLED_CONFIG_PATH)));
       }
       if (configs.hasPath(AUTH_SUCCESS_REDIRECT_PATH_CONFIG_PATH)) {
-        _authSuccessRedirectPath = configs.getString(AUTH_SUCCESS_REDIRECT_PATH_CONFIG_PATH);
+        authSuccessRedirectPath = configs.getString(AUTH_SUCCESS_REDIRECT_PATH_CONFIG_PATH);
       }
       return this;
     }
 
     public Builder from(String ssoSettingsJsonStr) {
       try {
-        jsonNode = _objectMapper.readTree(ssoSettingsJsonStr);
+        jsonNode = objectMapper.readTree(ssoSettingsJsonStr);
       } catch (Exception e) {
         throw new RuntimeException(
             String.format("Failed to parse ssoSettingsJsonStr %s into JSON", ssoSettingsJsonStr));
       }
       if (jsonNode.has(BASE_URL)) {
-        _authBaseUrl = jsonNode.get(BASE_URL).asText();
+        authBaseUrl = jsonNode.get(BASE_URL).asText();
       }
       if (jsonNode.has(OIDC_ENABLED)) {
-        _oidcEnabled = jsonNode.get(OIDC_ENABLED).asBoolean();
+        oidcEnabled = jsonNode.get(OIDC_ENABLED).asBoolean();
       }
 
       return this;
diff --git a/datahub-frontend/app/auth/sso/SsoManager.java b/datahub-frontend/app/auth/sso/SsoManager.java
index 8377eb40e237f..8a8a7f95a1b60 100644
--- a/datahub-frontend/app/auth/sso/SsoManager.java
+++ b/datahub-frontend/app/auth/sso/SsoManager.java
@@ -26,22 +26,21 @@
 public class SsoManager {
 
   private SsoProvider<?> _provider; // Only one active provider at a time.
-  private final Authentication
-      _authentication; // Authentication used to fetch SSO settings from GMS
-  private final String _ssoSettingsRequestUrl; // SSO settings request URL.
-  private final CloseableHttpClient _httpClient; // HTTP client for making requests to GMS.
-  private com.typesafe.config.Config _configs;
+  private final Authentication authentication; // Authentication used to fetch SSO settings from GMS
+  private final String ssoSettingsRequestUrl; // SSO settings request URL.
+  private final CloseableHttpClient httpClient; // HTTP client for making requests to GMS.
+  private com.typesafe.config.Config configs;
 
   public SsoManager(
       com.typesafe.config.Config configs,
       Authentication authentication,
       String ssoSettingsRequestUrl,
       CloseableHttpClient httpClient) {
-    _configs = configs;
-    _authentication = Objects.requireNonNull(authentication, "authentication cannot be null");
-    _ssoSettingsRequestUrl =
+    this.configs = configs;
+    this.authentication = Objects.requireNonNull(authentication, "authentication cannot be null");
+    this.ssoSettingsRequestUrl =
         Objects.requireNonNull(ssoSettingsRequestUrl, "ssoSettingsRequestUrl cannot be null");
-    _httpClient = Objects.requireNonNull(httpClient, "httpClient cannot be null");
+    this.httpClient = Objects.requireNonNull(httpClient, "httpClient cannot be null");
     _provider = null;
   }
 
@@ -52,6 +51,9 @@ public SsoManager(
    * @return true if SSO logic is enabled, false otherwise.
    */
   public boolean isSsoEnabled() {
+    if (configs.hasPath("auth.oidc.enabled") && configs.getBoolean("auth.oidc.enabled")) {
+      return true;
+    }
     refreshSsoProvider();
     return _provider != null;
   }
@@ -66,7 +68,7 @@ public void setSsoProvider(final SsoProvider<?> provider) {
   }
 
   public void setConfigs(final com.typesafe.config.Config configs) {
-    _configs = configs;
+    this.configs = configs;
   }
 
   public void clearSsoProvider() {
@@ -87,19 +89,19 @@ public SsoProvider<?> getSsoProvider() {
   public void initializeSsoProvider() {
     SsoConfigs ssoConfigs = null;
     try {
-      ssoConfigs = new SsoConfigs.Builder().from(_configs).build();
+      ssoConfigs = new SsoConfigs.Builder().from(configs).build();
     } catch (Exception e) {
       // Debug-level logging since this is expected to fail if SSO has not been configured.
-      log.debug(String.format("Missing SSO settings in static configs %s", _configs), e);
+      log.debug(String.format("Missing SSO settings in static configs %s", configs), e);
     }
 
     if (ssoConfigs != null && ssoConfigs.isOidcEnabled()) {
       try {
-        OidcConfigs oidcConfigs = new OidcConfigs.Builder().from(_configs).build();
+        OidcConfigs oidcConfigs = new OidcConfigs.Builder().from(configs).build();
         maybeUpdateOidcProvider(oidcConfigs);
       } catch (Exception e) {
         // Error-level logging since this is unexpected to fail if SSO has been configured.
-        log.error(String.format("Error building OidcConfigs from static configs %s", _configs), e);
+        log.error(String.format("Error building OidcConfigs from static configs %s", configs), e);
       }
     } else {
       // Clear the SSO Provider since no SSO is enabled.
@@ -132,7 +134,7 @@ private void refreshSsoProvider() {
     if (ssoConfigs != null && ssoConfigs.isOidcEnabled()) {
       try {
         OidcConfigs oidcConfigs =
-            new OidcConfigs.Builder().from(_configs, ssoSettingsJsonStr).build();
+            new OidcConfigs.Builder().from(configs, ssoSettingsJsonStr).build();
         maybeUpdateOidcProvider(oidcConfigs);
       } catch (Exception e) {
         log.error(
@@ -166,15 +168,15 @@ private void maybeUpdateOidcProvider(OidcConfigs oidcConfigs) {
   private Optional<String> getDynamicSsoSettings() {
     CloseableHttpResponse response = null;
     try {
-      final HttpPost request = new HttpPost(_ssoSettingsRequestUrl);
+      final HttpPost request = new HttpPost(ssoSettingsRequestUrl);
 
       // Build JSON request to verify credentials for a native user.
       request.setEntity(new StringEntity(""));
 
       // Add authorization header with DataHub frontend system id and secret.
-      request.addHeader(Http.HeaderNames.AUTHORIZATION, _authentication.getCredentials());
+      request.addHeader(Http.HeaderNames.AUTHORIZATION, authentication.getCredentials());
 
-      response = _httpClient.execute(request);
+      response = httpClient.execute(request);
       final HttpEntity entity = response.getEntity();
       if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && entity != null) {
         // Successfully received the SSO settings
diff --git a/datahub-frontend/app/auth/sso/SsoProvider.java b/datahub-frontend/app/auth/sso/SsoProvider.java
index a0947b52b92ae..2a6fff728966e 100644
--- a/datahub-frontend/app/auth/sso/SsoProvider.java
+++ b/datahub-frontend/app/auth/sso/SsoProvider.java
@@ -1,7 +1,6 @@
 package auth.sso;
 
 import org.pac4j.core.client.Client;
-import org.pac4j.core.credentials.Credentials;
 
 /** A thin interface over a Pac4j {@link Client} object and its associated configurations. */
 public interface SsoProvider<C extends SsoConfigs> {
@@ -12,14 +11,14 @@ enum SsoProtocol {
     // SAML -- not yet supported.
 
     // Common name appears in the Callback URL itself.
-    private final String _commonName;
+    private final String commonName;
 
     public String getCommonName() {
-      return _commonName;
+      return commonName;
     }
 
     SsoProtocol(String commonName) {
-      _commonName = commonName;
+      this.commonName = commonName;
     }
   }
 
@@ -30,5 +29,5 @@ public String getCommonName() {
   SsoProtocol protocol();
 
   /** Retrieves an initialized Pac4j {@link Client}. */
-  Client<? extends Credentials> client();
+  Client client();
 }
diff --git a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java
index fa676d2d16c90..3a4433b0ca81e 100644
--- a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java
+++ b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java
@@ -5,7 +5,7 @@
 import java.util.Map.Entry;
 import java.util.Optional;
 import org.pac4j.core.authorization.generator.AuthorizationGenerator;
-import org.pac4j.core.context.WebContext;
+import org.pac4j.core.context.CallContext;
 import org.pac4j.core.profile.AttributeLocation;
 import org.pac4j.core.profile.CommonProfile;
 import org.pac4j.core.profile.UserProfile;
@@ -18,24 +18,31 @@ public class OidcAuthorizationGenerator implements AuthorizationGenerator {
 
   private static final Logger logger = LoggerFactory.getLogger(OidcAuthorizationGenerator.class);
 
-  private final ProfileDefinition<?> profileDef;
-
+  private final ProfileDefinition profileDef;
   private final OidcConfigs oidcConfigs;
 
   public OidcAuthorizationGenerator(
-      final ProfileDefinition<?> profileDef, final OidcConfigs oidcConfigs) {
+      final ProfileDefinition profileDef, final OidcConfigs oidcConfigs) {
     this.profileDef = profileDef;
     this.oidcConfigs = oidcConfigs;
   }
 
   @Override
-  public Optional<UserProfile> generate(WebContext context, UserProfile profile) {
+  public Optional<UserProfile> generate(final CallContext context, final UserProfile profile) {
+    if (!(profile instanceof OidcProfile oidcProfile)) {
+      return Optional.of(profile);
+    }
+
     if (oidcConfigs.getExtractJwtAccessTokenClaims().orElse(false)) {
       try {
-        final JWT jwt = JWTParser.parse(((OidcProfile) profile).getAccessToken().getValue());
+        final JWT jwt = JWTParser.parse(oidcProfile.getAccessToken().getValue());
 
         CommonProfile commonProfile = new CommonProfile();
 
+        // Copy existing attributes
+        profile.getAttributes().forEach(commonProfile::addAttribute);
+
+        // Add JWT claims
         for (final Entry<String, Object> entry : jwt.getJWTClaimsSet().getClaims().entrySet()) {
           final String claimName = entry.getKey();
 
@@ -51,6 +58,6 @@ public Optional<UserProfile> generate(WebContext context, UserProfile profile) {
       }
     }
 
-    return Optional.ofNullable(profile);
+    return Optional.of(profile);
   }
 }
diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java
index 510804ba17f1a..113aeeb36551f 100644
--- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java
+++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java
@@ -39,6 +39,8 @@
 import com.linkedin.metadata.utils.GenericRecordUtils;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.r2.RemoteInvocationException;
+import com.linkedin.util.Pair;
+import io.datahubproject.metadata.context.OperationContext;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URI;
@@ -51,27 +53,36 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-
-import io.datahubproject.metadata.context.OperationContext;
+import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
+import org.pac4j.core.client.BaseClient;
+import org.pac4j.core.client.Client;
+import org.pac4j.core.client.Clients;
 import org.pac4j.core.config.Config;
+import org.pac4j.core.context.CallContext;
 import org.pac4j.core.context.Cookie;
+import org.pac4j.core.context.FrameworkParameters;
+import org.pac4j.core.context.WebContext;
+import org.pac4j.core.credentials.Credentials;
 import org.pac4j.core.engine.DefaultCallbackLogic;
+import org.pac4j.core.exception.http.HttpAction;
 import org.pac4j.core.http.adapter.HttpActionAdapter;
 import org.pac4j.core.profile.CommonProfile;
 import org.pac4j.core.profile.ProfileManager;
 import org.pac4j.core.profile.UserProfile;
+import org.pac4j.core.util.CommonHelper;
 import org.pac4j.core.util.Pac4jConstants;
-import org.pac4j.play.PlayWebContext;
+import org.pac4j.play.store.PlayCookieSessionStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import play.mvc.Result;
 
-import javax.annotation.Nonnull;
-
 /**
  * This class contains the logic that is executed when an OpenID Connect Identity Provider redirects
  * back to D DataHub after an authentication attempt.
@@ -83,7 +94,8 @@
  * if the user does not already exist.
  */
 @Slf4j
-public class OidcCallbackLogic extends DefaultCallbackLogic<Result, PlayWebContext> {
+public class OidcCallbackLogic extends DefaultCallbackLogic {
+  private static final Logger LOGGER = LoggerFactory.getLogger(OidcCallbackLogic.class);
 
   private final SsoManager ssoManager;
   private final SystemEntityClient systemEntityClient;
@@ -105,67 +117,130 @@ public OidcCallbackLogic(
   }
 
   @Override
-  public Result perform(
-      PlayWebContext context,
+  public Object perform(
       Config config,
-      HttpActionAdapter<Result, PlayWebContext> httpActionAdapter,
-      String defaultUrl,
-      Boolean saveInSession,
-      Boolean multiProfile,
-      Boolean renewSession,
-      String defaultClient) {
-
-    setContextRedirectUrl(context);
-
-    final Result result =
-        super.perform(
-            context,
-            config,
-            httpActionAdapter,
-            defaultUrl,
-            saveInSession,
-            multiProfile,
-            renewSession,
-            defaultClient);
+      String inputDefaultUrl,
+      Boolean inputRenewSession,
+      String defaultClient,
+      FrameworkParameters parameters) {
+
+    final Pair<CallContext, Object> ctxResult =
+        superPerform(config, inputDefaultUrl, inputRenewSession, defaultClient, parameters);
+
+    CallContext ctx = ctxResult.getFirst();
+    Result result = (Result) ctxResult.getSecond();
 
     // Handle OIDC authentication errors.
-    if (OidcResponseErrorHandler.isError(context)) {
-      return OidcResponseErrorHandler.handleError(context);
+    if (OidcResponseErrorHandler.isError(ctx)) {
+      return OidcResponseErrorHandler.handleError(ctx);
     }
 
     // By this point, we know that OIDC is the enabled provider.
     final OidcConfigs oidcConfigs = (OidcConfigs) ssoManager.getSsoProvider().configs();
-    return handleOidcCallback(systemOperationContext, oidcConfigs, result, getProfileManager(context));
+    return handleOidcCallback(systemOperationContext, ctx, oidcConfigs, result);
+  }
+
+  /** Overriding this to be able to intercept the CallContext being created */
+  private Pair<CallContext, Object> superPerform(
+      Config config,
+      String inputDefaultUrl,
+      Boolean inputRenewSession,
+      String defaultClient,
+      FrameworkParameters parameters) {
+    LOGGER.debug("=== CALLBACK ===");
+    CallContext ctx = this.buildContext(config, parameters);
+    WebContext webContext = ctx.webContext();
+    HttpActionAdapter httpActionAdapter = config.getHttpActionAdapter();
+    CommonHelper.assertNotNull("httpActionAdapter", httpActionAdapter);
+
+    HttpAction action;
+    try {
+      CommonHelper.assertNotNull("clientFinder", getClientFinder());
+      String defaultUrl = (String) Objects.requireNonNullElse(inputDefaultUrl, "/");
+      boolean renewSession = inputRenewSession == null || inputRenewSession;
+      CommonHelper.assertNotBlank("defaultUrl", defaultUrl);
+      Clients clients = config.getClients();
+      CommonHelper.assertNotNull("clients", clients);
+      List<Client> foundClients = getClientFinder().find(clients, webContext, defaultClient);
+      CommonHelper.assertTrue(
+          foundClients != null && foundClients.size() == 1,
+          "unable to find one indirect client for the callback: check the callback URL for a client name parameter or suffix path or ensure that your configuration defaults to one indirect client");
+      Client foundClient = (Client) foundClients.get(0);
+      LOGGER.debug("foundClient: {}", foundClient);
+      CommonHelper.assertNotNull("foundClient", foundClient);
+      Credentials credentials = (Credentials) foundClient.getCredentials(ctx).orElse(null);
+      LOGGER.debug("extracted credentials: {}", credentials);
+      credentials = (Credentials) foundClient.validateCredentials(ctx, credentials).orElse(null);
+      LOGGER.debug("validated credentials: {}", credentials);
+      if (credentials != null && !credentials.isForAuthentication()) {
+        action = foundClient.processLogout(ctx, credentials);
+      } else {
+        if (credentials != null) {
+          Optional<UserProfile> optProfile = foundClient.getUserProfile(ctx, credentials);
+          LOGGER.debug("optProfile: {}", optProfile);
+          if (optProfile.isPresent()) {
+            UserProfile profile = (UserProfile) optProfile.get();
+            Boolean saveProfileInSession =
+                ((BaseClient) foundClient).getSaveProfileInSession(webContext, profile);
+            boolean multiProfile = ((BaseClient) foundClient).isMultiProfile(webContext, profile);
+            LOGGER.debug(
+                "saveProfileInSession: {} / multiProfile: {}", saveProfileInSession, multiProfile);
+            this.saveUserProfile(
+                ctx, config, profile, saveProfileInSession, multiProfile, renewSession);
+          }
+        }
+
+        // Set the redirect url from cookie before creating action
+        setContextRedirectUrl(ctx);
+
+        action = this.redirectToOriginallyRequestedUrl(ctx, defaultUrl);
+      }
+    } catch (RuntimeException var20) {
+      RuntimeException e = var20;
+      return Pair.of(ctx, this.handleException(e, httpActionAdapter, webContext));
+    }
+
+    return Pair.of(ctx, httpActionAdapter.adapt(action, webContext));
   }
 
-  @SuppressWarnings("unchecked")
-  private void setContextRedirectUrl(PlayWebContext context) {
+  private void setContextRedirectUrl(CallContext ctx) {
+    WebContext context = ctx.webContext();
+    PlayCookieSessionStore sessionStore = (PlayCookieSessionStore) ctx.sessionStore();
+
     Optional<Cookie> redirectUrl =
         context.getRequestCookies().stream()
             .filter(cookie -> REDIRECT_URL_COOKIE_NAME.equals(cookie.getName()))
             .findFirst();
     redirectUrl.ifPresent(
         cookie ->
-            context
-                .getSessionStore()
-                .set(
-                    context,
-                    Pac4jConstants.REQUESTED_URL,
-                    JAVA_SER_HELPER.deserializeFromBytes(
+            sessionStore.set(
+                context,
+                Pac4jConstants.REQUESTED_URL,
+                sessionStore
+                    .getSerializer()
+                    .deserializeFromBytes(
                         uncompressBytes(Base64.getDecoder().decode(cookie.getValue())))));
   }
 
   private Result handleOidcCallback(
       final OperationContext opContext,
+      final CallContext ctx,
       final OidcConfigs oidcConfigs,
-      final Result result,
-      final ProfileManager<UserProfile> profileManager) {
+      final Result result) {
 
     log.debug("Beginning OIDC Callback Handling...");
 
+    ProfileManager profileManager =
+        ctx.profileManagerFactory().apply(ctx.webContext(), ctx.sessionStore());
+
     if (profileManager.isAuthenticated()) {
       // If authenticated, the user should have a profile.
-      final CommonProfile profile = (CommonProfile) profileManager.get(true).get();
+      final Optional<UserProfile> optProfile = profileManager.getProfile();
+      if (optProfile.isEmpty()) {
+        return internalServerError(
+            "Failed to authenticate current user. Cannot find valid identity provider profile in session.");
+      }
+      final CommonProfile profile = (CommonProfile) optProfile.get();
       log.debug(
           String.format(
               "Found authenticated user with profile %s", profile.getAttributes().toString()));
@@ -196,7 +271,8 @@ private Result handleOidcCallback(
         }
         // Update user status to active on login.
         // If we want to prevent certain users from logging in, here's where we'll want to do it.
-        setUserStatus(opContext,
+        setUserStatus(
+            opContext,
             corpUserUrn,
             new CorpUserStatus()
                 .setStatus(Constants.CORP_USER_STATUS_ACTIVE)
@@ -307,29 +383,32 @@ private CorpUserSnapshot extractUser(CorpuserUrn urn, CommonProfile profile) {
     return corpUserSnapshot;
   }
 
-  public static Collection<String> getGroupNames(CommonProfile profile, Object groupAttribute, String groupsClaimName) {
-      Collection<String> groupNames = Collections.emptyList();
-      try {
-        if (groupAttribute instanceof Collection) {
-          // List of group names
-          groupNames = (Collection<String>) profile.getAttribute(groupsClaimName, Collection.class);
-        } else if (groupAttribute instanceof String) {
-          String groupString = (String) groupAttribute;
-          ObjectMapper objectMapper = new ObjectMapper();
-          try {
-            // Json list of group names
-            groupNames = objectMapper.readValue(groupString, new TypeReference<List<String>>(){});
-          } catch (Exception e) {
-            groupNames = Arrays.asList(groupString.split(","));
-          }
+  public static Collection<String> getGroupNames(
+      CommonProfile profile, Object groupAttribute, String groupsClaimName) {
+    Collection<String> groupNames = Collections.emptyList();
+    try {
+      if (groupAttribute instanceof Collection) {
+        // List of group names
+        groupNames = (Collection<String>) profile.getAttribute(groupsClaimName, Collection.class);
+      } else if (groupAttribute instanceof String) {
+        String groupString = (String) groupAttribute;
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+          // Json list of group names
+          groupNames = objectMapper.readValue(groupString, new TypeReference<List<String>>() {});
+        } catch (Exception e) {
+          groupNames = Arrays.asList(groupString.split(","));
         }
-      } catch (Exception e) {
-        log.error(String.format(
-                "Failed to parse group names: Expected to find a list of strings for attribute with name %s, found %s",
-                groupsClaimName, profile.getAttribute(groupsClaimName).getClass()));
       }
-      return groupNames;
+    } catch (Exception e) {
+      log.error(
+          String.format(
+              "Failed to parse group names: Expected to find a list of strings for attribute with name %s, found %s",
+              groupsClaimName, profile.getAttribute(groupsClaimName).getClass()));
+    }
+    return groupNames;
   }
+
   private List<CorpGroupSnapshot> extractGroups(CommonProfile profile) {
 
     log.debug(
@@ -350,7 +429,8 @@ private List<CorpGroupSnapshot> extractGroups(CommonProfile profile) {
       if (profile.containsAttribute(groupsClaimName)) {
         try {
           final List<CorpGroupSnapshot> groupSnapshots = new ArrayList<>();
-          Collection<String> groupNames = getGroupNames(profile, profile.getAttribute(groupsClaimName), groupsClaimName);
+          Collection<String> groupNames =
+              getGroupNames(profile, profile.getAttribute(groupsClaimName), groupsClaimName);
 
           for (String groupName : groupNames) {
             // Create a basic CorpGroupSnapshot from the information.
@@ -405,7 +485,8 @@ private GroupMembership createGroupMembership(final List<CorpGroupSnapshot> extr
     return groupMembershipAspect;
   }
 
-  private void tryProvisionUser(@Nonnull OperationContext opContext, CorpUserSnapshot corpUserSnapshot) {
+  private void tryProvisionUser(
+      @Nonnull OperationContext opContext, CorpUserSnapshot corpUserSnapshot) {
 
     log.debug(String.format("Attempting to provision user with urn %s", corpUserSnapshot.getUrn()));
 
@@ -439,7 +520,8 @@ private void tryProvisionUser(@Nonnull OperationContext opContext, CorpUserSnaps
     }
   }
 
-  private void tryProvisionGroups(@Nonnull OperationContext opContext, List<CorpGroupSnapshot> corpGroups) {
+  private void tryProvisionGroups(
+      @Nonnull OperationContext opContext, List<CorpGroupSnapshot> corpGroups) {
 
     log.debug(
         String.format(
@@ -450,8 +532,7 @@ private void tryProvisionGroups(@Nonnull OperationContext opContext, List<CorpGr
     try {
       final Set<Urn> urnsToFetch =
           corpGroups.stream().map(CorpGroupSnapshot::getUrn).collect(Collectors.toSet());
-      final Map<Urn, Entity> existingGroups =
-          systemEntityClient.batchGet(opContext, urnsToFetch);
+      final Map<Urn, Entity> existingGroups = systemEntityClient.batchGet(opContext, urnsToFetch);
 
       log.debug(String.format("Fetched GMS groups with urns %s", existingGroups.keySet()));
 
@@ -489,7 +570,8 @@ private void tryProvisionGroups(@Nonnull OperationContext opContext, List<CorpGr
       log.debug(String.format("Provisioning groups with urns %s", groupsToCreateUrns));
 
       // Now batch create all entities identified to create.
-      systemEntityClient.batchUpdate(opContext,
+      systemEntityClient.batchUpdate(
+          opContext,
           groupsToCreate.stream()
               .map(groupSnapshot -> new Entity().setValue(Snapshot.create(groupSnapshot)))
               .collect(Collectors.toSet()));
@@ -505,7 +587,8 @@ private void tryProvisionGroups(@Nonnull OperationContext opContext, List<CorpGr
     }
   }
 
-  private void updateGroupMembership(@Nonnull OperationContext opContext, Urn urn, GroupMembership groupMembership) {
+  private void updateGroupMembership(
+      @Nonnull OperationContext opContext, Urn urn, GroupMembership groupMembership) {
     log.debug(String.format("Updating group membership for user %s", urn));
     final MetadataChangeProposal proposal = new MetadataChangeProposal();
     proposal.setEntityUrn(urn);
@@ -547,7 +630,9 @@ private void verifyPreProvisionedUser(@Nonnull OperationContext opContext, Corpu
     }
   }
 
-  private void setUserStatus(@Nonnull OperationContext opContext, final Urn urn, final CorpUserStatus newStatus) throws Exception {
+  private void setUserStatus(
+      @Nonnull OperationContext opContext, final Urn urn, final CorpUserStatus newStatus)
+      throws Exception {
     // Update status aspect to be active.
     final MetadataChangeProposal proposal = new MetadataChangeProposal();
     proposal.setEntityUrn(urn);
diff --git a/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java b/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java
index f982944071498..61c1269b76858 100644
--- a/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java
+++ b/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java
@@ -38,6 +38,7 @@ public class OidcConfigs extends SsoConfigs {
   public static final String OIDC_USE_NONCE = "auth.oidc.useNonce";
   public static final String OIDC_CUSTOM_PARAM_RESOURCE = "auth.oidc.customParam.resource";
   public static final String OIDC_READ_TIMEOUT = "auth.oidc.readTimeout";
+  public static final String OIDC_CONNECT_TIMEOUT = "auth.oidc.connectTimeout";
   public static final String OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS =
       "auth.oidc.extractJwtAccessTokenClaims";
   public static final String OIDC_PREFERRED_JWS_ALGORITHM = "auth.oidc.preferredJwsAlgorithm";
@@ -58,6 +59,7 @@ public class OidcConfigs extends SsoConfigs {
   // False since extraction of groups can overwrite existing group membership.
   private static final String DEFAULT_OIDC_GROUPS_CLAIM = "groups";
   private static final String DEFAULT_OIDC_READ_TIMEOUT = "5000";
+  private static final String DEFAULT_OIDC_CONNECT_TIMEOUT = "1000";
 
   private final String clientId;
   private final String clientSecret;
@@ -76,6 +78,7 @@ public class OidcConfigs extends SsoConfigs {
   private final Optional<Boolean> useNonce;
   private final Optional<String> customParamResource;
   private final String readTimeout;
+  private final String connectTimeout;
   private final Optional<Boolean> extractJwtAccessTokenClaims;
   private final Optional<String> preferredJwsAlgorithm;
   private final Optional<String> grantType;
@@ -100,6 +103,7 @@ public OidcConfigs(Builder builder) {
     this.useNonce = builder.useNonce;
     this.customParamResource = builder.customParamResource;
     this.readTimeout = builder.readTimeout;
+    this.connectTimeout = builder.connectTimeout;
     this.extractJwtAccessTokenClaims = builder.extractJwtAccessTokenClaims;
     this.preferredJwsAlgorithm = builder.preferredJwsAlgorithm;
     this.acrValues = builder.acrValues;
@@ -127,6 +131,7 @@ public static class Builder extends SsoConfigs.Builder<Builder> {
     private Optional<Boolean> useNonce = Optional.empty();
     private Optional<String> customParamResource = Optional.empty();
     private String readTimeout = DEFAULT_OIDC_READ_TIMEOUT;
+    private String connectTimeout = DEFAULT_OIDC_CONNECT_TIMEOUT;
     private Optional<Boolean> extractJwtAccessTokenClaims = Optional.empty();
     private Optional<String> preferredJwsAlgorithm = Optional.empty();
     private Optional<String> grantType = Optional.empty();
@@ -173,6 +178,7 @@ public Builder from(final com.typesafe.config.Config configs) {
       useNonce = getOptional(configs, OIDC_USE_NONCE).map(Boolean::parseBoolean);
       customParamResource = getOptional(configs, OIDC_CUSTOM_PARAM_RESOURCE);
       readTimeout = getOptional(configs, OIDC_READ_TIMEOUT, DEFAULT_OIDC_READ_TIMEOUT);
+      connectTimeout = getOptional(configs, OIDC_CONNECT_TIMEOUT, DEFAULT_OIDC_CONNECT_TIMEOUT);
       extractJwtAccessTokenClaims =
           getOptional(configs, OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).map(Boolean::parseBoolean);
       preferredJwsAlgorithm =
@@ -232,6 +238,9 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe
       if (jsonNode.has(READ_TIMEOUT)) {
         readTimeout = jsonNode.get(READ_TIMEOUT).asText();
       }
+      if (jsonNode.has(CONNECT_TIMEOUT)) {
+        connectTimeout = jsonNode.get(CONNECT_TIMEOUT).asText();
+      }
       if (jsonNode.has(EXTRACT_JWT_ACCESS_TOKEN_CLAIMS)) {
         extractJwtAccessTokenClaims =
             Optional.of(jsonNode.get(EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).asBoolean());
@@ -250,11 +259,11 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe
     }
 
     public OidcConfigs build() {
-      Objects.requireNonNull(_oidcEnabled, "oidcEnabled is required");
+      Objects.requireNonNull(oidcEnabled, "oidcEnabled is required");
       Objects.requireNonNull(clientId, "clientId is required");
       Objects.requireNonNull(clientSecret, "clientSecret is required");
       Objects.requireNonNull(discoveryUri, "discoveryUri is required");
-      Objects.requireNonNull(_authBaseUrl, "authBaseUrl is required");
+      Objects.requireNonNull(authBaseUrl, "authBaseUrl is required");
 
       return new OidcConfigs(this);
     }
diff --git a/datahub-frontend/app/auth/sso/oidc/OidcProvider.java b/datahub-frontend/app/auth/sso/oidc/OidcProvider.java
index a8a3205e8299c..7fcaa5a9683cb 100644
--- a/datahub-frontend/app/auth/sso/oidc/OidcProvider.java
+++ b/datahub-frontend/app/auth/sso/oidc/OidcProvider.java
@@ -2,15 +2,16 @@
 
 import auth.sso.SsoProvider;
 import auth.sso.oidc.custom.CustomOidcClient;
-import com.google.common.collect.ImmutableMap;
+import com.nimbusds.jose.JWSAlgorithm;
 import java.util.HashMap;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.pac4j.core.client.Client;
 import org.pac4j.core.http.callback.PathParameterCallbackUrlResolver;
 import org.pac4j.oidc.config.OidcConfiguration;
-import org.pac4j.oidc.credentials.OidcCredentials;
 import org.pac4j.oidc.profile.OidcProfileDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Implementation of {@link SsoProvider} supporting the OIDC protocol.
@@ -24,25 +25,25 @@
  */
 @Slf4j
 public class OidcProvider implements SsoProvider<OidcConfigs> {
-
+  private static final Logger logger = LoggerFactory.getLogger(OidcProvider.class);
   private static final String OIDC_CLIENT_NAME = "oidc";
 
-  private final OidcConfigs _oidcConfigs;
-  private final Client<OidcCredentials> _oidcClient; // Used primarily for redirecting to IdP.
+  private final OidcConfigs oidcConfigs;
+  private final Client oidcClient; // Used primarily for redirecting to IdP.
 
   public OidcProvider(final OidcConfigs configs) {
-    _oidcConfigs = configs;
-    _oidcClient = createPac4jClient();
+    oidcConfigs = configs;
+    oidcClient = createPac4jClient();
   }
 
   @Override
-  public Client<OidcCredentials> client() {
-    return _oidcClient;
+  public Client client() {
+    return oidcClient;
   }
 
   @Override
   public OidcConfigs configs() {
-    return _oidcConfigs;
+    return oidcConfigs;
   }
 
   @Override
@@ -50,50 +51,52 @@ public SsoProtocol protocol() {
     return SsoProtocol.OIDC;
   }
 
-  private Client<OidcCredentials> createPac4jClient() {
+  private Client createPac4jClient() {
     final OidcConfiguration oidcConfiguration = new OidcConfiguration();
-    oidcConfiguration.setClientId(_oidcConfigs.getClientId());
-    oidcConfiguration.setSecret(_oidcConfigs.getClientSecret());
-    oidcConfiguration.setDiscoveryURI(_oidcConfigs.getDiscoveryUri());
+    oidcConfiguration.setClientId(oidcConfigs.getClientId());
+    oidcConfiguration.setSecret(oidcConfigs.getClientSecret());
+    oidcConfiguration.setDiscoveryURI(oidcConfigs.getDiscoveryUri());
     oidcConfiguration.setClientAuthenticationMethodAsString(
-        _oidcConfigs.getClientAuthenticationMethod());
-    oidcConfiguration.setScope(_oidcConfigs.getScope());
+        oidcConfigs.getClientAuthenticationMethod());
+    oidcConfiguration.setScope(oidcConfigs.getScope());
     try {
-      oidcConfiguration.setReadTimeout(Integer.parseInt(_oidcConfigs.getReadTimeout()));
+      oidcConfiguration.setConnectTimeout(Integer.parseInt(oidcConfigs.getConnectTimeout()));
+    } catch (NumberFormatException e) {
+      log.warn("Invalid connect timeout configuration, defaulting to 1000ms");
+    }
+    try {
+      oidcConfiguration.setReadTimeout(Integer.parseInt(oidcConfigs.getReadTimeout()));
     } catch (NumberFormatException e) {
       log.warn("Invalid read timeout configuration, defaulting to 5000ms");
     }
-    _oidcConfigs.getResponseType().ifPresent(oidcConfiguration::setResponseType);
-    _oidcConfigs.getResponseMode().ifPresent(oidcConfiguration::setResponseMode);
-    _oidcConfigs.getUseNonce().ifPresent(oidcConfiguration::setUseNonce);
+    oidcConfigs.getResponseType().ifPresent(oidcConfiguration::setResponseType);
+    oidcConfigs.getResponseMode().ifPresent(oidcConfiguration::setResponseMode);
+    oidcConfigs.getUseNonce().ifPresent(oidcConfiguration::setUseNonce);
     Map<String, String> customParamsMap = new HashMap<>();
-    _oidcConfigs
-        .getCustomParamResource()
-        .ifPresent(value -> customParamsMap.put("resource", value));
-    _oidcConfigs
-        .getGrantType()
-        .ifPresent(value -> customParamsMap.put("grant_type", value));
-    _oidcConfigs
-        .getAcrValues()
-        .ifPresent(value -> customParamsMap.put("acr_values", value));
+    oidcConfigs.getCustomParamResource().ifPresent(value -> customParamsMap.put("resource", value));
+    oidcConfigs.getGrantType().ifPresent(value -> customParamsMap.put("grant_type", value));
+    oidcConfigs.getAcrValues().ifPresent(value -> customParamsMap.put("acr_values", value));
     if (!customParamsMap.isEmpty()) {
       oidcConfiguration.setCustomParams(customParamsMap);
     }
-    _oidcConfigs
+    oidcConfigs
         .getPreferredJwsAlgorithm()
         .ifPresent(
             preferred -> {
               log.info("Setting preferredJwsAlgorithm: " + preferred);
-              oidcConfiguration.setPreferredJwsAlgorithm(preferred);
+              oidcConfiguration.setPreferredJwsAlgorithm(JWSAlgorithm.parse(preferred));
             });
 
+    // Enable state parameter validation
+    oidcConfiguration.setWithState(true);
+
     final CustomOidcClient oidcClient = new CustomOidcClient(oidcConfiguration);
     oidcClient.setName(OIDC_CLIENT_NAME);
-    oidcClient.setCallbackUrl(
-        _oidcConfigs.getAuthBaseUrl() + _oidcConfigs.getAuthBaseCallbackPath());
+    oidcClient.setCallbackUrl(oidcConfigs.getAuthBaseUrl() + oidcConfigs.getAuthBaseCallbackPath());
     oidcClient.setCallbackUrlResolver(new PathParameterCallbackUrlResolver());
     oidcClient.addAuthorizationGenerator(
-        new OidcAuthorizationGenerator(new OidcProfileDefinition(), _oidcConfigs));
+        new OidcAuthorizationGenerator(new OidcProfileDefinition(), oidcConfigs));
+
     return oidcClient;
   }
 }
diff --git a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java
index 9881b5e095b78..2843beee61610 100644
--- a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java
+++ b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java
@@ -4,7 +4,8 @@
 import static play.mvc.Results.unauthorized;
 
 import java.util.Optional;
-import org.pac4j.play.PlayWebContext;
+import org.pac4j.core.context.CallContext;
+import org.pac4j.core.context.WebContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import play.mvc.Result;
@@ -13,14 +14,14 @@ public class OidcResponseErrorHandler {
 
   private OidcResponseErrorHandler() {}
 
-  private static final Logger _logger = LoggerFactory.getLogger("OidcResponseErrorHandler");
+  private static final Logger logger = LoggerFactory.getLogger("OidcResponseErrorHandler");
 
   private static final String ERROR_FIELD_NAME = "error";
   private static final String ERROR_DESCRIPTION_FIELD_NAME = "error_description";
 
-  public static Result handleError(final PlayWebContext context) {
-
-    _logger.warn(
+  public static Result handleError(final CallContext ctx) {
+    WebContext context = ctx.webContext();
+    logger.warn(
         "OIDC responded with an error: '{}'. Error description: '{}'",
         getError(context),
         getErrorDescription(context));
@@ -44,15 +45,15 @@ public static Result handleError(final PlayWebContext context) {
             getError(context).orElse(""), getErrorDescription(context).orElse("")));
   }
 
-  public static boolean isError(final PlayWebContext context) {
-    return getError(context).isPresent() && !getError(context).get().isEmpty();
+  public static boolean isError(final CallContext ctx) {
+    return getError(ctx.webContext()).isPresent() && !getError(ctx.webContext()).get().isEmpty();
   }
 
-  public static Optional<String> getError(final PlayWebContext context) {
+  public static Optional<String> getError(final WebContext context) {
     return context.getRequestParameter(ERROR_FIELD_NAME);
   }
 
-  public static Optional<String> getErrorDescription(final PlayWebContext context) {
+  public static Optional<String> getErrorDescription(final WebContext context) {
     return context.getRequestParameter(ERROR_DESCRIPTION_FIELD_NAME);
   }
 }
diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java
index 01f8f16171d13..2288547cf6ed1 100644
--- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java
+++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java
@@ -2,7 +2,6 @@
 
 import com.nimbusds.oauth2.sdk.AuthorizationCode;
 import com.nimbusds.oauth2.sdk.AuthorizationCodeGrant;
-import com.nimbusds.oauth2.sdk.AuthorizationGrant;
 import com.nimbusds.oauth2.sdk.ParseException;
 import com.nimbusds.oauth2.sdk.TokenErrorResponse;
 import com.nimbusds.oauth2.sdk.TokenRequest;
@@ -18,6 +17,7 @@
 import com.nimbusds.oauth2.sdk.pkce.CodeVerifier;
 import com.nimbusds.openid.connect.sdk.OIDCTokenResponse;
 import com.nimbusds.openid.connect.sdk.OIDCTokenResponseParser;
+import com.nimbusds.openid.connect.sdk.op.OIDCProviderMetadata;
 import com.nimbusds.openid.connect.sdk.token.OIDCTokens;
 import java.io.IOException;
 import java.net.URI;
@@ -25,9 +25,11 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
+import org.pac4j.core.context.CallContext;
 import org.pac4j.core.context.WebContext;
-import org.pac4j.core.credentials.authenticator.Authenticator;
+import org.pac4j.core.credentials.Credentials;
 import org.pac4j.core.exception.TechnicalException;
 import org.pac4j.core.util.CommonHelper;
 import org.pac4j.oidc.client.OidcClient;
@@ -37,9 +39,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class CustomOidcAuthenticator implements Authenticator<OidcCredentials> {
+public class CustomOidcAuthenticator extends OidcAuthenticator {
 
-  private static final Logger logger = LoggerFactory.getLogger(OidcAuthenticator.class);
+  private static final Logger logger = LoggerFactory.getLogger(CustomOidcAuthenticator.class);
 
   private static final Collection<ClientAuthenticationMethod> SUPPORTED_METHODS =
       Arrays.asList(
@@ -47,21 +49,24 @@ public class CustomOidcAuthenticator implements Authenticator<OidcCredentials> {
           ClientAuthenticationMethod.CLIENT_SECRET_BASIC,
           ClientAuthenticationMethod.NONE);
 
-  protected OidcConfiguration configuration;
-
-  protected OidcClient<OidcConfiguration> client;
-
   private final ClientAuthentication clientAuthentication;
 
-  public CustomOidcAuthenticator(final OidcClient<OidcConfiguration> client) {
-    CommonHelper.assertNotNull("configuration", client.getConfiguration());
-    CommonHelper.assertNotNull("client", client);
-    this.configuration = client.getConfiguration();
-    this.client = client;
+  public CustomOidcAuthenticator(final OidcClient client) {
+    super(client.getConfiguration(), client);
 
     // check authentication methods
-    final List<ClientAuthenticationMethod> metadataMethods =
-        configuration.findProviderMetadata().getTokenEndpointAuthMethods();
+    OIDCProviderMetadata providerMetadata;
+    try {
+      providerMetadata = loadWithRetry();
+    } catch (TechnicalException e) {
+      logger.error(
+          "Could not resolve identity provider's remote configuration from DiscoveryURI: {}",
+          configuration.getDiscoveryURI());
+      throw e;
+    }
+
+    List<ClientAuthenticationMethod> metadataMethods =
+        providerMetadata.getTokenEndpointAuthMethods();
 
     final ClientAuthenticationMethod preferredMethod =
         getPreferredAuthenticationMethod(configuration);
@@ -146,8 +151,11 @@ private static ClientAuthenticationMethod firstSupportedMethod(
   }
 
   @Override
-  public void validate(final OidcCredentials credentials, final WebContext context) {
-    final AuthorizationCode code = credentials.getCode();
+  public Optional<Credentials> validate(CallContext ctx, Credentials cred) {
+    OidcCredentials credentials = (OidcCredentials) cred;
+    WebContext context = ctx.webContext();
+
+    final AuthorizationCode code = credentials.toAuthorizationCode();
     // if we have a code
     if (code != null) {
       try {
@@ -156,7 +164,7 @@ public void validate(final OidcCredentials credentials, final WebContext context
             (CodeVerifier)
                 configuration
                     .getValueRetriever()
-                    .retrieve(client.getCodeVerifierSessionAttributeName(), client, context)
+                    .retrieve(ctx, client.getCodeVerifierSessionAttributeName(), client)
                     .orElse(null);
         // Token request
         final TokenRequest request =
@@ -182,27 +190,49 @@ public void validate(final OidcCredentials credentials, final WebContext context
 
         // save tokens in credentials
         final OIDCTokens oidcTokens = tokenSuccessResponse.getOIDCTokens();
-        credentials.setAccessToken(oidcTokens.getAccessToken());
-        credentials.setRefreshToken(oidcTokens.getRefreshToken());
-        credentials.setIdToken(oidcTokens.getIDToken());
+        credentials.setAccessTokenObject(oidcTokens.getAccessToken());
+
+        // Only set refresh token if it exists
+        if (oidcTokens.getRefreshToken() != null) {
+          credentials.setRefreshTokenObject(oidcTokens.getRefreshToken());
+        }
+
+        if (oidcTokens.getIDToken() != null) {
+          credentials.setIdToken(oidcTokens.getIDToken().getParsedString());
+        }
 
       } catch (final URISyntaxException | IOException | ParseException e) {
         throw new TechnicalException(e);
       }
     }
+
+    return Optional.ofNullable(cred);
   }
 
-  private TokenRequest createTokenRequest(final AuthorizationGrant grant) {
-    if (clientAuthentication != null) {
-      return new TokenRequest(
-          configuration.findProviderMetadata().getTokenEndpointURI(),
-          this.clientAuthentication,
-          grant);
-    } else {
-      return new TokenRequest(
-          configuration.findProviderMetadata().getTokenEndpointURI(),
-          new ClientID(configuration.getClientId()),
-          grant);
+  // Simple retry with exponential backoff
+  public OIDCProviderMetadata loadWithRetry() {
+    int maxAttempts = 3;
+    long initialDelay = 1000; // 1 second
+
+    for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        OIDCProviderMetadata providerMetadata = configuration.getOpMetadataResolver().load();
+        return Objects.requireNonNull(providerMetadata);
+      } catch (RuntimeException e) {
+        if (attempt == maxAttempts) {
+          throw e; // Rethrow on final attempt
+        }
+        try {
+          // Exponential backoff
+          Thread.sleep(initialDelay * (long) Math.pow(2, attempt - 1));
+        } catch (InterruptedException ie) {
+          Thread.currentThread().interrupt();
+          throw new RuntimeException("Retry interrupted", ie);
+        }
+        logger.warn("Retry attempt {} of {} failed", attempt, maxAttempts, e);
+      }
     }
+    throw new RuntimeException(
+        "Failed to load provider metadata after " + maxAttempts + " attempts");
   }
 }
diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java
index 3a0a247cb761e..28c927c2aa541 100644
--- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java
+++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java
@@ -3,25 +3,38 @@
 import org.pac4j.core.util.CommonHelper;
 import org.pac4j.oidc.client.OidcClient;
 import org.pac4j.oidc.config.OidcConfiguration;
-import org.pac4j.oidc.credentials.extractor.OidcExtractor;
+import org.pac4j.oidc.credentials.extractor.OidcCredentialsExtractor;
 import org.pac4j.oidc.logout.OidcLogoutActionBuilder;
+import org.pac4j.oidc.logout.processor.OidcLogoutProcessor;
 import org.pac4j.oidc.profile.creator.OidcProfileCreator;
-import org.pac4j.oidc.redirect.OidcRedirectionActionBuilder;
 
-public class CustomOidcClient extends OidcClient<OidcConfiguration> {
+public class CustomOidcClient extends OidcClient {
 
-  public CustomOidcClient(final OidcConfiguration configuration) {
-    setConfiguration(configuration);
+  public CustomOidcClient(OidcConfiguration configuration) {
+    super(configuration);
   }
 
   @Override
-  protected void clientInit() {
+  protected void internalInit(final boolean forceReinit) {
+    // Validate configuration
     CommonHelper.assertNotNull("configuration", getConfiguration());
-    getConfiguration().init();
-    defaultRedirectionActionBuilder(new CustomOidcRedirectionActionBuilder(getConfiguration(), this));
-    defaultCredentialsExtractor(new OidcExtractor(getConfiguration(), this));
-    defaultAuthenticator(new CustomOidcAuthenticator(this));
-    defaultProfileCreator(new OidcProfileCreator<>(getConfiguration(), this));
-    defaultLogoutActionBuilder(new OidcLogoutActionBuilder(getConfiguration()));
+
+    // Initialize configuration
+    getConfiguration().init(forceReinit);
+
+    // Initialize client components
+    setRedirectionActionBuilderIfUndefined(
+        new CustomOidcRedirectionActionBuilder(getConfiguration(), this));
+    setCredentialsExtractorIfUndefined(new OidcCredentialsExtractor(getConfiguration(), this));
+
+    // Initialize default authenticator if not set
+    if (getAuthenticator() == null || forceReinit) {
+      setAuthenticatorIfUndefined(new CustomOidcAuthenticator(this));
+    }
+
+    setProfileCreatorIfUndefined(new OidcProfileCreator(getConfiguration(), this));
+    setLogoutProcessorIfUndefined(
+        new OidcLogoutProcessor(getConfiguration(), findSessionLogoutHandler()));
+    setLogoutActionBuilderIfUndefined(new OidcLogoutActionBuilder(getConfiguration()));
   }
 }
diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java
index bdeeacc895af3..ea5315972344d 100644
--- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java
+++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java
@@ -2,29 +2,35 @@
 
 import java.util.Map;
 import java.util.Optional;
+import org.pac4j.core.context.CallContext;
 import org.pac4j.core.context.WebContext;
 import org.pac4j.core.exception.http.RedirectionAction;
-import org.pac4j.core.exception.http.RedirectionActionHelper;
+import org.pac4j.core.util.HttpActionHelper;
 import org.pac4j.oidc.client.OidcClient;
 import org.pac4j.oidc.config.OidcConfiguration;
 import org.pac4j.oidc.redirect.OidcRedirectionActionBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
 public class CustomOidcRedirectionActionBuilder extends OidcRedirectionActionBuilder {
 
   private static final Logger logger = LoggerFactory.getLogger(OidcRedirectionActionBuilder.class);
+
+  private final OidcConfiguration configuration;
+
   public CustomOidcRedirectionActionBuilder(OidcConfiguration configuration, OidcClient client) {
-    super(configuration, client);
+    super(client);
+    this.configuration = configuration;
   }
 
   @Override
-  public Optional<RedirectionAction> getRedirectionAction(WebContext context) {
-    Map<String, String> params = this.buildParams();
+  public Optional<RedirectionAction> getRedirectionAction(CallContext ctx) {
+    WebContext context = ctx.webContext();
+
+    Map<String, String> params = this.buildParams(context);
     String computedCallbackUrl = this.client.computeFinalCallbackUrl(context);
     params.put("redirect_uri", computedCallbackUrl);
-    this.addStateAndNonceParameters(context, params);
+    this.addStateAndNonceParameters(ctx, params);
     if (this.configuration.getMaxAge() != null) {
       params.put("max_age", this.configuration.getMaxAge().toString());
     }
@@ -40,7 +46,6 @@ public Optional<RedirectionAction> getRedirectionAction(WebContext context) {
     }
 
     logger.debug("Authentication request url: {}", location);
-    return Optional.of(RedirectionActionHelper.buildRedirectUrlAction(context, location));
+    return Optional.of(HttpActionHelper.buildRedirectUrlAction(context, location));
   }
-
 }
diff --git a/datahub-frontend/app/client/KafkaTrackingProducer.java b/datahub-frontend/app/client/KafkaTrackingProducer.java
index 058e75100c24a..a29fe3bb7aef7 100644
--- a/datahub-frontend/app/client/KafkaTrackingProducer.java
+++ b/datahub-frontend/app/client/KafkaTrackingProducer.java
@@ -27,7 +27,8 @@
 
 @Singleton
 public class KafkaTrackingProducer {
-  private final Logger _logger = LoggerFactory.getLogger(KafkaTrackingProducer.class.getName());
+  private static final Logger logger =
+      LoggerFactory.getLogger(KafkaTrackingProducer.class.getName());
   private static final List<String> KAFKA_SSL_PROTOCOLS =
       Collections.unmodifiableList(
           Arrays.asList(
@@ -35,38 +36,38 @@ public class KafkaTrackingProducer {
               SecurityProtocol.SASL_SSL.name(),
               SecurityProtocol.SASL_PLAINTEXT.name()));
 
-  private final Boolean _isEnabled;
-  private final KafkaProducer<String, String> _producer;
+  private final Boolean isEnabled;
+  private final KafkaProducer<String, String> producer;
 
   @Inject
   public KafkaTrackingProducer(
       @Nonnull Config config,
       ApplicationLifecycle lifecycle,
       final ConfigurationProvider configurationProvider) {
-    _isEnabled = !config.hasPath("analytics.enabled") || config.getBoolean("analytics.enabled");
+    isEnabled = !config.hasPath("analytics.enabled") || config.getBoolean("analytics.enabled");
 
-    if (_isEnabled) {
-      _logger.debug("Analytics tracking is enabled");
-      _producer = createKafkaProducer(config, configurationProvider.getKafka());
+    if (isEnabled) {
+      logger.debug("Analytics tracking is enabled");
+      producer = createKafkaProducer(config, configurationProvider.getKafka());
 
       lifecycle.addStopHook(
           () -> {
-            _producer.flush();
-            _producer.close();
+            producer.flush();
+            producer.close();
             return CompletableFuture.completedFuture(null);
           });
     } else {
-      _logger.debug("Analytics tracking is disabled");
-      _producer = null;
+      logger.debug("Analytics tracking is disabled");
+      producer = null;
     }
   }
 
   public Boolean isEnabled() {
-    return _isEnabled;
+    return isEnabled;
   }
 
   public void send(ProducerRecord<String, String> record) {
-    _producer.send(record);
+    producer.send(record);
   }
 
   private static KafkaProducer createKafkaProducer(
diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java
index d447b28cdcc46..97e916769a6c4 100644
--- a/datahub-frontend/app/config/ConfigurationProvider.java
+++ b/datahub-frontend/app/config/ConfigurationProvider.java
@@ -6,12 +6,9 @@
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import lombok.Data;
-import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.PropertySource;
-import org.springframework.stereotype.Component;
 
 /**
  * Minimal sharing between metadata-service and frontend Does not use the factories module to avoid
diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java
index 017847367de05..1c3786f4af526 100644
--- a/datahub-frontend/app/controllers/Application.java
+++ b/datahub-frontend/app/controllers/Application.java
@@ -36,8 +36,8 @@
 import play.libs.ws.StandaloneWSClient;
 import play.libs.ws.ahc.StandaloneAhcWSClient;
 import play.mvc.Controller;
-import play.mvc.Http.Cookie;
 import play.mvc.Http;
+import play.mvc.Http.Cookie;
 import play.mvc.ResponseHeader;
 import play.mvc.Result;
 import play.mvc.Security;
@@ -48,16 +48,16 @@
 import utils.ConfigUtil;
 
 public class Application extends Controller {
-  private final Logger _logger = LoggerFactory.getLogger(Application.class.getName());
-  private final Config _config;
-  private final StandaloneWSClient _ws;
-  private final Environment _environment;
+  private static final Logger logger = LoggerFactory.getLogger(Application.class.getName());
+  private final Config config;
+  private final StandaloneWSClient ws;
+  private final Environment environment;
 
   @Inject
   public Application(Environment environment, @Nonnull Config config) {
-    _config = config;
-    _ws = createWsClient();
-    _environment = environment;
+    this.config = config;
+    ws = createWsClient();
+    this.environment = environment;
   }
 
   /**
@@ -69,10 +69,10 @@ public Application(Environment environment, @Nonnull Config config) {
   @Nonnull
   private Result serveAsset(@Nullable String path) {
     try {
-      InputStream indexHtml = _environment.resourceAsStream("public/index.html");
+      InputStream indexHtml = environment.resourceAsStream("public/index.html");
       return ok(indexHtml).withHeader("Cache-Control", "no-cache").as("text/html");
     } catch (Exception e) {
-      _logger.warn("Cannot load public/index.html resource. Static assets or assets jar missing?");
+      logger.warn("Cannot load public/index.html resource. Static assets or assets jar missing?");
       return notFound().withHeader("Cache-Control", "no-cache").as("text/html");
     }
   }
@@ -106,17 +106,17 @@ public CompletableFuture<Result> proxy(String path, Http.Request request)
 
     final String metadataServiceHost =
         ConfigUtil.getString(
-            _config,
+            config,
             ConfigUtil.METADATA_SERVICE_HOST_CONFIG_PATH,
             ConfigUtil.DEFAULT_METADATA_SERVICE_HOST);
     final int metadataServicePort =
         ConfigUtil.getInt(
-            _config,
+            config,
             ConfigUtil.METADATA_SERVICE_PORT_CONFIG_PATH,
             ConfigUtil.DEFAULT_METADATA_SERVICE_PORT);
     final boolean metadataServiceUseSsl =
         ConfigUtil.getBoolean(
-            _config,
+            config,
             ConfigUtil.METADATA_SERVICE_USE_SSL_CONFIG_PATH,
             ConfigUtil.DEFAULT_METADATA_SERVICE_USE_SSL);
 
@@ -139,7 +139,7 @@ public CompletableFuture<Result> proxy(String path, Http.Request request)
     // Get the current time to measure the duration of the request
     Instant start = Instant.now();
 
-    return _ws.url(
+    return ws.url(
             String.format(
                 "%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri))
         .setMethod(request.method())
@@ -167,9 +167,10 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req
         .execute()
         .thenApply(
             apiResponse -> {
-              // Log the query if it takes longer than the configured threshold and verbose logging is enabled
-              boolean verboseGraphQLLogging = _config.getBoolean("graphql.verbose.logging");
-              int verboseGraphQLLongQueryMillis = _config.getInt("graphql.verbose.slowQueryMillis");
+              // Log the query if it takes longer than the configured threshold and verbose logging
+              // is enabled
+              boolean verboseGraphQLLogging = config.getBoolean("graphql.verbose.logging");
+              int verboseGraphQLLongQueryMillis = config.getInt("graphql.verbose.slowQueryMillis");
               Instant finish = Instant.now();
               long timeElapsed = Duration.between(start, finish).toMillis();
               if (verboseGraphQLLogging && timeElapsed >= verboseGraphQLLongQueryMillis) {
@@ -206,32 +207,32 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req
   public Result appConfig() {
     final ObjectNode config = Json.newObject();
     config.put("application", "datahub-frontend");
-    config.put("appVersion", _config.getString("app.version"));
-    config.put("isInternal", _config.getBoolean("linkedin.internal"));
-    config.put("shouldShowDatasetLineage", _config.getBoolean("linkedin.show.dataset.lineage"));
+    config.put("appVersion", this.config.getString("app.version"));
+    config.put("isInternal", this.config.getBoolean("linkedin.internal"));
+    config.put("shouldShowDatasetLineage", this.config.getBoolean("linkedin.show.dataset.lineage"));
     config.put(
         "suggestionConfidenceThreshold",
-        Integer.valueOf(_config.getString("linkedin.suggestion.confidence.threshold")));
+        Integer.valueOf(this.config.getString("linkedin.suggestion.confidence.threshold")));
     config.set("wikiLinks", wikiLinks());
     config.set("tracking", trackingInfo());
     // In a staging environment, we can trigger this flag to be true so that the UI can handle based
     // on
     // such config and alert users that their changes will not affect production data
-    config.put("isStagingBanner", _config.getBoolean("ui.show.staging.banner"));
-    config.put("isLiveDataWarning", _config.getBoolean("ui.show.live.data.banner"));
-    config.put("showChangeManagement", _config.getBoolean("ui.show.CM.banner"));
+    config.put("isStagingBanner", this.config.getBoolean("ui.show.staging.banner"));
+    config.put("isLiveDataWarning", this.config.getBoolean("ui.show.live.data.banner"));
+    config.put("showChangeManagement", this.config.getBoolean("ui.show.CM.banner"));
     // Flag to enable people entity elements
-    config.put("showPeople", _config.getBoolean("ui.show.people"));
-    config.put("changeManagementLink", _config.getString("ui.show.CM.link"));
+    config.put("showPeople", this.config.getBoolean("ui.show.people"));
+    config.put("changeManagementLink", this.config.getString("ui.show.CM.link"));
     // Flag set in order to warn users that search is experiencing issues
-    config.put("isStaleSearch", _config.getBoolean("ui.show.stale.search"));
-    config.put("showAdvancedSearch", _config.getBoolean("ui.show.advanced.search"));
+    config.put("isStaleSearch", this.config.getBoolean("ui.show.stale.search"));
+    config.put("showAdvancedSearch", this.config.getBoolean("ui.show.advanced.search"));
     // Flag to use the new api for browsing datasets
-    config.put("useNewBrowseDataset", _config.getBoolean("ui.new.browse.dataset"));
+    config.put("useNewBrowseDataset", this.config.getBoolean("ui.new.browse.dataset"));
     // show lineage graph in relationships tabs
-    config.put("showLineageGraph", _config.getBoolean("ui.show.lineage.graph"));
+    config.put("showLineageGraph", this.config.getBoolean("ui.show.lineage.graph"));
     // show institutional memory for available entities
-    config.put("showInstitutionalMemory", _config.getBoolean("ui.show.institutional.memory"));
+    config.put("showInstitutionalMemory", this.config.getBoolean("ui.show.institutional.memory"));
 
     // Insert properties for user profile operations
     config.set("userEntityProps", userEntityProps());
@@ -250,8 +251,8 @@ public Result appConfig() {
   @Nonnull
   private ObjectNode userEntityProps() {
     final ObjectNode props = Json.newObject();
-    props.put("aviUrlPrimary", _config.getString("linkedin.links.avi.urlPrimary"));
-    props.put("aviUrlFallback", _config.getString("linkedin.links.avi.urlFallback"));
+    props.put("aviUrlPrimary", config.getString("linkedin.links.avi.urlPrimary"));
+    props.put("aviUrlFallback", config.getString("linkedin.links.avi.urlFallback"));
     return props;
   }
 
@@ -261,19 +262,19 @@ private ObjectNode userEntityProps() {
   @Nonnull
   private ObjectNode wikiLinks() {
     final ObjectNode wikiLinks = Json.newObject();
-    wikiLinks.put("appHelp", _config.getString("links.wiki.appHelp"));
-    wikiLinks.put("gdprPii", _config.getString("links.wiki.gdprPii"));
-    wikiLinks.put("tmsSchema", _config.getString("links.wiki.tmsSchema"));
-    wikiLinks.put("gdprTaxonomy", _config.getString("links.wiki.gdprTaxonomy"));
-    wikiLinks.put("staleSearchIndex", _config.getString("links.wiki.staleSearchIndex"));
-    wikiLinks.put("dht", _config.getString("links.wiki.dht"));
-    wikiLinks.put("purgePolicies", _config.getString("links.wiki.purgePolicies"));
-    wikiLinks.put("jitAcl", _config.getString("links.wiki.jitAcl"));
-    wikiLinks.put("metadataCustomRegex", _config.getString("links.wiki.metadataCustomRegex"));
-    wikiLinks.put("exportPolicy", _config.getString("links.wiki.exportPolicy"));
-    wikiLinks.put("metadataHealth", _config.getString("links.wiki.metadataHealth"));
-    wikiLinks.put("purgeKey", _config.getString("links.wiki.purgeKey"));
-    wikiLinks.put("datasetDecommission", _config.getString("links.wiki.datasetDecommission"));
+    wikiLinks.put("appHelp", config.getString("links.wiki.appHelp"));
+    wikiLinks.put("gdprPii", config.getString("links.wiki.gdprPii"));
+    wikiLinks.put("tmsSchema", config.getString("links.wiki.tmsSchema"));
+    wikiLinks.put("gdprTaxonomy", config.getString("links.wiki.gdprTaxonomy"));
+    wikiLinks.put("staleSearchIndex", config.getString("links.wiki.staleSearchIndex"));
+    wikiLinks.put("dht", config.getString("links.wiki.dht"));
+    wikiLinks.put("purgePolicies", config.getString("links.wiki.purgePolicies"));
+    wikiLinks.put("jitAcl", config.getString("links.wiki.jitAcl"));
+    wikiLinks.put("metadataCustomRegex", config.getString("links.wiki.metadataCustomRegex"));
+    wikiLinks.put("exportPolicy", config.getString("links.wiki.exportPolicy"));
+    wikiLinks.put("metadataHealth", config.getString("links.wiki.metadataHealth"));
+    wikiLinks.put("purgeKey", config.getString("links.wiki.purgeKey"));
+    wikiLinks.put("datasetDecommission", config.getString("links.wiki.datasetDecommission"));
     return wikiLinks;
   }
 
@@ -283,8 +284,8 @@ private ObjectNode wikiLinks() {
   @Nonnull
   private ObjectNode trackingInfo() {
     final ObjectNode piwik = Json.newObject();
-    piwik.put("piwikSiteId", Integer.valueOf(_config.getString("tracking.piwik.siteid")));
-    piwik.put("piwikUrl", _config.getString("tracking.piwik.url"));
+    piwik.put("piwikSiteId", Integer.valueOf(config.getString("tracking.piwik.siteid")));
+    piwik.put("piwikUrl", config.getString("tracking.piwik.url"));
 
     final ObjectNode trackers = Json.newObject();
     trackers.set("piwik", piwik);
@@ -376,9 +377,10 @@ private String mapPath(@Nonnull final String path) {
     return path;
   }
 
-
   /**
-   * Called if verbose logging is enabled and request takes longer that the slow query milliseconds defined in the config
+   * Called if verbose logging is enabled and request takes longer that the slow query milliseconds
+   * defined in the config
+   *
    * @param request GraphQL request that was made
    * @param resolvedUri URI that was requested
    * @param duration How long the query took to complete
@@ -393,16 +395,16 @@ private void logSlowQuery(Http.Request request, String resolvedUri, float durati
       JsonNode jsonNode = request.body().asJson();
       ((ObjectNode) jsonNode).remove("query");
       jsonBody.append(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonNode));
-    }
-    catch (Exception e) {
-      _logger.info("GraphQL Request Received: {}, Unable to parse JSON body", resolvedUri);
+    } catch (Exception e) {
+      logger.info("GraphQL Request Received: {}, Unable to parse JSON body", resolvedUri);
     }
     String jsonBodyStr = jsonBody.toString();
-    _logger.info("Slow GraphQL Request Received: {}, Request query string: {}, Request actor: {}, Request JSON: {}, Request completed in {} ms",
-            resolvedUri,
-            request.queryString(),
-            actorValue,
-            jsonBodyStr,
-            duration);
+    logger.info(
+        "Slow GraphQL Request Received: {}, Request query string: {}, Request actor: {}, Request JSON: {}, Request completed in {} ms",
+        resolvedUri,
+        request.queryString(),
+        actorValue,
+        jsonBodyStr,
+        duration);
   }
 }
diff --git a/datahub-frontend/app/controllers/AuthenticationController.java b/datahub-frontend/app/controllers/AuthenticationController.java
index 87c4b5ba06793..3c8ace9aee29f 100644
--- a/datahub-frontend/app/controllers/AuthenticationController.java
+++ b/datahub-frontend/app/controllers/AuthenticationController.java
@@ -26,12 +26,14 @@
 import org.apache.commons.httpclient.InvalidRedirectLocationException;
 import org.apache.commons.lang3.StringUtils;
 import org.pac4j.core.client.Client;
+import org.pac4j.core.context.CallContext;
 import org.pac4j.core.context.Cookie;
+import org.pac4j.core.context.WebContext;
 import org.pac4j.core.exception.http.FoundAction;
 import org.pac4j.core.exception.http.RedirectionAction;
 import org.pac4j.play.PlayWebContext;
 import org.pac4j.play.http.PlayHttpActionAdapter;
-import org.pac4j.play.store.PlaySessionStore;
+import org.pac4j.play.store.PlayCookieSessionStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import play.data.validation.Constraints;
@@ -51,26 +53,27 @@ public class AuthenticationController extends Controller {
   private static final String SSO_NO_REDIRECT_MESSAGE =
       "SSO is configured, however missing redirect from idp";
 
-  private final Logger _logger = LoggerFactory.getLogger(AuthenticationController.class.getName());
-  private final CookieConfigs _cookieConfigs;
-  private final JAASConfigs _jaasConfigs;
-  private final NativeAuthenticationConfigs _nativeAuthenticationConfigs;
-  private final boolean _verbose;
+  private static final Logger logger =
+      LoggerFactory.getLogger(AuthenticationController.class.getName());
+  private final CookieConfigs cookieConfigs;
+  private final JAASConfigs jaasConfigs;
+  private final NativeAuthenticationConfigs nativeAuthenticationConfigs;
+  private final boolean verbose;
 
-  @Inject private org.pac4j.core.config.Config _ssoConfig;
+  @Inject private org.pac4j.core.config.Config ssoConfig;
 
-  @Inject private PlaySessionStore _playSessionStore;
+  @Inject private PlayCookieSessionStore playCookieSessionStore;
 
-  @Inject private SsoManager _ssoManager;
+  @Inject private SsoManager ssoManager;
 
-  @Inject AuthServiceClient _authClient;
+  @Inject AuthServiceClient authClient;
 
   @Inject
   public AuthenticationController(@Nonnull Config configs) {
-    _cookieConfigs = new CookieConfigs(configs);
-    _jaasConfigs = new JAASConfigs(configs);
-    _nativeAuthenticationConfigs = new NativeAuthenticationConfigs(configs);
-    _verbose = configs.hasPath(AUTH_VERBOSE_LOGGING) && configs.getBoolean(AUTH_VERBOSE_LOGGING);
+    cookieConfigs = new CookieConfigs(configs);
+    jaasConfigs = new JAASConfigs(configs);
+    nativeAuthenticationConfigs = new NativeAuthenticationConfigs(configs);
+    verbose = configs.hasPath(AUTH_VERBOSE_LOGGING) && configs.getBoolean(AUTH_VERBOSE_LOGGING);
   }
 
   /**
@@ -92,11 +95,14 @@ public Result authenticate(Http.Request request) {
     try {
       URI redirectUri = new URI(redirectPath);
       if (redirectUri.getScheme() != null || redirectUri.getAuthority() != null) {
-        throw new InvalidRedirectLocationException("Redirect location must be relative to the base url, cannot "
-            + "redirect to other domains: " + redirectPath, redirectPath);
+        throw new InvalidRedirectLocationException(
+            "Redirect location must be relative to the base url, cannot "
+                + "redirect to other domains: "
+                + redirectPath,
+            redirectPath);
       }
     } catch (URISyntaxException | InvalidRedirectLocationException e) {
-      _logger.warn(e.getMessage());
+      logger.warn(e.getMessage());
       redirectPath = "/";
     }
 
@@ -105,7 +111,7 @@ public Result authenticate(Http.Request request) {
     }
 
     // 1. If SSO is enabled, redirect to IdP if not authenticated.
-    if (_ssoManager.isSsoEnabled()) {
+    if (ssoManager.isSsoEnabled()) {
       return redirectToIdentityProvider(request, redirectPath)
           .orElse(
               Results.redirect(
@@ -114,8 +120,8 @@ public Result authenticate(Http.Request request) {
     }
 
     // 2. If either JAAS auth or Native auth is enabled, fallback to it
-    if (_jaasConfigs.isJAASEnabled()
-        || _nativeAuthenticationConfigs.isNativeAuthenticationEnabled()) {
+    if (jaasConfigs.isJAASEnabled()
+        || nativeAuthenticationConfigs.isNativeAuthenticationEnabled()) {
       return Results.redirect(
           LOGIN_ROUTE
               + String.format("?%s=%s", AUTH_REDIRECT_URI_PARAM, encodeRedirectUri(redirectPath)));
@@ -123,21 +129,21 @@ public Result authenticate(Http.Request request) {
 
     // 3. If no auth enabled, fallback to using default user account & redirect.
     // Generate GMS session token, TODO:
-    final String accessToken = _authClient.generateSessionTokenForUser(DEFAULT_ACTOR_URN.getId());
+    final String accessToken = authClient.generateSessionTokenForUser(DEFAULT_ACTOR_URN.getId());
     return Results.redirect(redirectPath)
         .withSession(createSessionMap(DEFAULT_ACTOR_URN.toString(), accessToken))
         .withCookies(
             createActorCookie(
                 DEFAULT_ACTOR_URN.toString(),
-                _cookieConfigs.getTtlInHours(),
-                _cookieConfigs.getAuthCookieSameSite(),
-                _cookieConfigs.getAuthCookieSecure()));
+                cookieConfigs.getTtlInHours(),
+                cookieConfigs.getAuthCookieSameSite(),
+                cookieConfigs.getAuthCookieSecure()));
   }
 
   /** Redirect to the identity provider for authentication. */
   @Nonnull
   public Result sso(Http.Request request) {
-    if (_ssoManager.isSsoEnabled()) {
+    if (ssoManager.isSsoEnabled()) {
       return redirectToIdentityProvider(request, "/")
           .orElse(
               Results.redirect(
@@ -156,11 +162,11 @@ public Result sso(Http.Request request) {
    */
   @Nonnull
   public Result logIn(Http.Request request) {
-    boolean jaasEnabled = _jaasConfigs.isJAASEnabled();
-    _logger.debug(String.format("Jaas authentication enabled: %b", jaasEnabled));
+    boolean jaasEnabled = jaasConfigs.isJAASEnabled();
+    logger.debug(String.format("Jaas authentication enabled: %b", jaasEnabled));
     boolean nativeAuthenticationEnabled =
-        _nativeAuthenticationConfigs.isNativeAuthenticationEnabled();
-    _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled));
+        nativeAuthenticationConfigs.isNativeAuthenticationEnabled();
+    logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled));
     boolean noAuthEnabled = !jaasEnabled && !nativeAuthenticationEnabled;
     if (noAuthEnabled) {
       String message = "Neither JAAS nor native authentication is enabled on the server.";
@@ -182,13 +188,13 @@ public Result logIn(Http.Request request) {
     boolean loginSucceeded = tryLogin(username, password);
 
     if (!loginSucceeded) {
-      _logger.info("Login failed for user: {}", username);
+      logger.info("Login failed for user: {}", username);
       return Results.badRequest(invalidCredsJson);
     }
 
     final Urn actorUrn = new CorpuserUrn(username);
-    _logger.info("Login successful for user: {}, urn: {}", username, actorUrn);
-    final String accessToken = _authClient.generateSessionTokenForUser(actorUrn.getId());
+    logger.info("Login successful for user: {}, urn: {}", username, actorUrn);
+    final String accessToken = authClient.generateSessionTokenForUser(actorUrn.getId());
     return createSession(actorUrn.toString(), accessToken);
   }
 
@@ -199,8 +205,8 @@ public Result logIn(Http.Request request) {
   @Nonnull
   public Result signUp(Http.Request request) {
     boolean nativeAuthenticationEnabled =
-        _nativeAuthenticationConfigs.isNativeAuthenticationEnabled();
-    _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled));
+        nativeAuthenticationConfigs.isNativeAuthenticationEnabled();
+    logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled));
     if (!nativeAuthenticationEnabled) {
       String message = "Native authentication is not enabled on the server.";
       final ObjectNode error = Json.newObject();
@@ -224,7 +230,7 @@ public Result signUp(Http.Request request) {
       JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty.");
       return Results.badRequest(invalidCredsJson);
     }
-    if (_nativeAuthenticationConfigs.isEnforceValidEmailEnabled()) {
+    if (nativeAuthenticationConfigs.isEnforceValidEmailEnabled()) {
       Constraints.EmailValidator emailValidator = new Constraints.EmailValidator();
       if (!emailValidator.isValid(email)) {
         JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty.");
@@ -250,9 +256,9 @@ public Result signUp(Http.Request request) {
 
     final Urn userUrn = new CorpuserUrn(email);
     final String userUrnString = userUrn.toString();
-    _authClient.signUp(userUrnString, fullName, email, title, password, inviteToken);
-    _logger.info("Signed up user {} using invite tokens", userUrnString);
-    final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId());
+    authClient.signUp(userUrnString, fullName, email, title, password, inviteToken);
+    logger.info("Signed up user {} using invite tokens", userUrnString);
+    final String accessToken = authClient.generateSessionTokenForUser(userUrn.getId());
     return createSession(userUrnString, accessToken);
   }
 
@@ -260,8 +266,8 @@ public Result signUp(Http.Request request) {
   @Nonnull
   public Result resetNativeUserCredentials(Http.Request request) {
     boolean nativeAuthenticationEnabled =
-        _nativeAuthenticationConfigs.isNativeAuthenticationEnabled();
-    _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled));
+        nativeAuthenticationConfigs.isNativeAuthenticationEnabled();
+    logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled));
     if (!nativeAuthenticationEnabled) {
       String message = "Native authentication is not enabled on the server.";
       final ObjectNode error = Json.newObject();
@@ -291,26 +297,27 @@ public Result resetNativeUserCredentials(Http.Request request) {
 
     final Urn userUrn = new CorpuserUrn(email);
     final String userUrnString = userUrn.toString();
-    _authClient.resetNativeUserCredentials(userUrnString, password, resetToken);
-    final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId());
+    authClient.resetNativeUserCredentials(userUrnString, password, resetToken);
+    final String accessToken = authClient.generateSessionTokenForUser(userUrn.getId());
     return createSession(userUrnString, accessToken);
   }
 
   private Optional<Result> redirectToIdentityProvider(
       Http.RequestHeader request, String redirectPath) {
-    final PlayWebContext playWebContext = new PlayWebContext(request, _playSessionStore);
-    final Client client = _ssoManager.getSsoProvider().client();
-    configurePac4jSessionStore(playWebContext, client, redirectPath);
+    CallContext ctx = buildCallContext(request);
+
+    final Client client = ssoManager.getSsoProvider().client();
+    configurePac4jSessionStore(ctx, client, redirectPath);
     try {
-      final Optional<RedirectionAction> action = client.getRedirectionAction(playWebContext);
-      return action.map(act -> new PlayHttpActionAdapter().adapt(act, playWebContext));
+      final Optional<RedirectionAction> action = client.getRedirectionAction(ctx);
+      return action.map(act -> new PlayHttpActionAdapter().adapt(act, ctx.webContext()));
     } catch (Exception e) {
-      if (_verbose) {
-        _logger.error(
+      if (verbose) {
+        logger.error(
             "Caught exception while attempting to redirect to SSO identity provider! It's likely that SSO integration is mis-configured",
             e);
       } else {
-        _logger.error(
+        logger.error(
             "Caught exception while attempting to redirect to SSO identity provider! It's likely that SSO integration is mis-configured");
       }
       return Optional.of(
@@ -324,22 +331,33 @@ private Optional<Result> redirectToIdentityProvider(
     }
   }
 
-  private void configurePac4jSessionStore(
-      PlayWebContext context, Client client, String redirectPath) {
+  private CallContext buildCallContext(Http.RequestHeader request) {
+    // First create PlayWebContext from the request
+    PlayWebContext webContext = new PlayWebContext(request);
+
+    // Then create CallContext using the web context and session store
+    return new CallContext(webContext, playCookieSessionStore);
+  }
+
+  private void configurePac4jSessionStore(CallContext ctx, Client client, String redirectPath) {
+    WebContext context = ctx.webContext();
+
     // Set the originally requested path for post-auth redirection. We split off into a separate
     // cookie from the session
     // to reduce size of the session cookie
     FoundAction foundAction = new FoundAction(redirectPath);
-    byte[] javaSerBytes = JAVA_SER_HELPER.serializeToBytes(foundAction);
+    byte[] javaSerBytes =
+        ((PlayCookieSessionStore) ctx.sessionStore()).getSerializer().serializeToBytes(foundAction);
     String serialized = Base64.getEncoder().encodeToString(compressBytes(javaSerBytes));
     context.addResponseCookie(new Cookie(REDIRECT_URL_COOKIE_NAME, serialized));
     // This is to prevent previous login attempts from being cached.
     // We replicate the logic here, which is buried in the Pac4j client.
-    if (_playSessionStore.get(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX)
-        != null) {
-      _logger.debug(
+    Optional<Object> attempt =
+        playCookieSessionStore.get(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX);
+    if (attempt.isPresent() && !"".equals(attempt.get())) {
+      logger.debug(
           "Found previous login attempt. Removing it manually to prevent unexpected errors.");
-      _playSessionStore.set(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX, "");
+      playCookieSessionStore.set(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX, "");
     }
   }
 
@@ -351,27 +369,27 @@ private boolean tryLogin(String username, String password) {
     boolean loginSucceeded = false;
 
     // First try jaas login, if enabled
-    if (_jaasConfigs.isJAASEnabled()) {
+    if (jaasConfigs.isJAASEnabled()) {
       try {
-        _logger.debug("Attempting JAAS authentication for user: {}", username);
+        logger.debug("Attempting JAAS authentication for user: {}", username);
         AuthenticationManager.authenticateJaasUser(username, password);
-        _logger.debug("JAAS authentication successful. Login succeeded");
+        logger.debug("JAAS authentication successful. Login succeeded");
         loginSucceeded = true;
       } catch (Exception e) {
-        if (_verbose) {
-          _logger.debug("JAAS authentication error. Login failed", e);
+        if (verbose) {
+          logger.debug("JAAS authentication error. Login failed", e);
         } else {
-          _logger.debug("JAAS authentication error. Login failed");
+          logger.debug("JAAS authentication error. Login failed");
         }
       }
     }
 
     // If jaas login fails or is disabled, try native auth login
-    if (_nativeAuthenticationConfigs.isNativeAuthenticationEnabled() && !loginSucceeded) {
+    if (nativeAuthenticationConfigs.isNativeAuthenticationEnabled() && !loginSucceeded) {
       final Urn userUrn = new CorpuserUrn(username);
       final String userUrnString = userUrn.toString();
       loginSucceeded =
-          loginSucceeded || _authClient.verifyNativeUserCredentials(userUrnString, password);
+          loginSucceeded || authClient.verifyNativeUserCredentials(userUrnString, password);
     }
 
     return loginSucceeded;
@@ -383,8 +401,8 @@ private Result createSession(String userUrnString, String accessToken) {
         .withCookies(
             createActorCookie(
                 userUrnString,
-                _cookieConfigs.getTtlInHours(),
-                _cookieConfigs.getAuthCookieSameSite(),
-                _cookieConfigs.getAuthCookieSecure()));
+                cookieConfigs.getTtlInHours(),
+                cookieConfigs.getAuthCookieSameSite(),
+                cookieConfigs.getAuthCookieSecure()));
   }
 }
diff --git a/datahub-frontend/app/controllers/CentralLogoutController.java b/datahub-frontend/app/controllers/CentralLogoutController.java
index eea1c662ebf89..d2e3027236adc 100644
--- a/datahub-frontend/app/controllers/CentralLogoutController.java
+++ b/datahub-frontend/app/controllers/CentralLogoutController.java
@@ -1,6 +1,6 @@
 package controllers;
 
-import com.typesafe.config.Config;
+import auth.sso.SsoManager;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import javax.inject.Inject;
@@ -15,13 +15,10 @@
 public class CentralLogoutController extends LogoutController {
   private static final String AUTH_URL_CONFIG_PATH = "/login";
   private static final String DEFAULT_BASE_URL_PATH = "/";
-  private static Boolean _isOidcEnabled = false;
+  @Inject private SsoManager ssoManager;
 
-  @Inject
-  public CentralLogoutController(Config config) {
-    _isOidcEnabled = config.hasPath("auth.oidc.enabled") && config.getBoolean("auth.oidc.enabled");
-
-    setDefaultUrl(DEFAULT_BASE_URL_PATH);
+  public CentralLogoutController() {
+    setDefaultUrl(AUTH_URL_CONFIG_PATH);
     setLogoutUrlPattern(DEFAULT_BASE_URL_PATH + ".*");
     setLocalLogout(true);
     setCentralLogout(true);
@@ -29,7 +26,7 @@ public CentralLogoutController(Config config) {
 
   /** logout() method should not be called if oidc is not enabled */
   public Result executeLogout(Http.Request request) {
-    if (_isOidcEnabled) {
+    if (ssoManager.isSsoEnabled()) {
       try {
         return logout(request).toCompletableFuture().get().withNewSession();
       } catch (Exception e) {
diff --git a/datahub-frontend/app/controllers/RedirectController.java b/datahub-frontend/app/controllers/RedirectController.java
index 17f86b7fbffae..a86584e24ca29 100644
--- a/datahub-frontend/app/controllers/RedirectController.java
+++ b/datahub-frontend/app/controllers/RedirectController.java
@@ -16,7 +16,10 @@ public Result favicon(Http.Request request) {
     if (config.getVisualConfig().getAssets().getFaviconUrl().startsWith("http")) {
       return permanentRedirect(config.getVisualConfig().getAssets().getFaviconUrl());
     } else {
-      final String prefix = config.getVisualConfig().getAssets().getFaviconUrl().startsWith("/") ? "/public" : "/public/";
+      final String prefix =
+          config.getVisualConfig().getAssets().getFaviconUrl().startsWith("/")
+              ? "/public"
+              : "/public/";
       return ok(Application.class.getResourceAsStream(
               prefix + config.getVisualConfig().getAssets().getFaviconUrl()))
           .as("image/x-icon");
diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java
index 750886570bf40..385b02a56ba23 100644
--- a/datahub-frontend/app/controllers/SsoCallbackController.java
+++ b/datahub-frontend/app/controllers/SsoCallbackController.java
@@ -5,8 +5,8 @@
 import auth.sso.SsoProvider;
 import auth.sso.oidc.OidcCallbackLogic;
 import client.AuthServiceClient;
-import com.datahub.authentication.Authentication;
 import com.linkedin.entity.client.SystemEntityClient;
+import io.datahubproject.metadata.context.OperationContext;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -16,16 +16,15 @@
 import javax.annotation.Nonnull;
 import javax.inject.Inject;
 import javax.inject.Named;
-
-import io.datahubproject.metadata.context.OperationContext;
 import lombok.extern.slf4j.Slf4j;
+import org.pac4j.core.adapter.FrameworkAdapter;
 import org.pac4j.core.client.Client;
 import org.pac4j.core.client.Clients;
 import org.pac4j.core.config.Config;
+import org.pac4j.core.context.FrameworkParameters;
 import org.pac4j.core.engine.CallbackLogic;
-import org.pac4j.core.http.adapter.HttpActionAdapter;
 import org.pac4j.play.CallbackController;
-import org.pac4j.play.PlayWebContext;
+import org.pac4j.play.context.PlayFrameworkParameters;
 import play.mvc.Http;
 import play.mvc.Result;
 import play.mvc.Results;
@@ -40,8 +39,9 @@
 @Slf4j
 public class SsoCallbackController extends CallbackController {
 
-  private final SsoManager _ssoManager;
-  private final Config _config;
+  private final SsoManager ssoManager;
+  private final Config config;
+  private final CallbackLogic callbackLogic;
 
   @Inject
   public SsoCallbackController(
@@ -51,23 +51,41 @@ public SsoCallbackController(
       @Nonnull AuthServiceClient authClient,
       @Nonnull Config config,
       @Nonnull com.typesafe.config.Config configs) {
-    _ssoManager = ssoManager;
-    _config = config;
+    this.ssoManager = ssoManager;
+    this.config = config;
     setDefaultUrl("/"); // By default, redirects to Home Page on log in.
-    setSaveInSession(false);
-    setCallbackLogic(
+
+    callbackLogic =
         new SsoCallbackLogic(
             ssoManager,
-                systemOperationContext,
+            systemOperationContext,
             entityClient,
             authClient,
-            new CookieConfigs(configs)));
+            new CookieConfigs(configs));
+  }
+
+  @Override
+  public CompletionStage<Result> callback(Http.Request request) {
+    FrameworkAdapter.INSTANCE.applyDefaultSettingsIfUndefined(this.config);
+
+    return CompletableFuture.supplyAsync(
+        () -> {
+          return (Result)
+              callbackLogic.perform(
+                  this.config,
+                  getDefaultUrl(),
+                  getRenewSession(),
+                  getDefaultClient(),
+                  new PlayFrameworkParameters(request));
+        },
+        this.ec.current());
   }
 
   public CompletionStage<Result> handleCallback(String protocol, Http.Request request) {
     if (shouldHandleCallback(protocol)) {
-      log.debug("Handling SSO callback. Protocol: {}",
-        _ssoManager.getSsoProvider().protocol().getCommonName());
+      log.debug(
+          "Handling SSO callback. Protocol: {}",
+          ssoManager.getSsoProvider().protocol().getCommonName());
       return callback(request)
           .handle(
               (res, e) -> {
@@ -94,9 +112,9 @@ public CompletionStage<Result> handleCallback(String protocol, Http.Request requ
   }
 
   /** Logic responsible for delegating to protocol-specific callback logic. */
-  public class SsoCallbackLogic implements CallbackLogic<Result, PlayWebContext> {
+  public class SsoCallbackLogic implements CallbackLogic {
 
-    private final OidcCallbackLogic _oidcCallbackLogic;
+    private final OidcCallbackLogic oidcCallbackLogic;
 
     SsoCallbackLogic(
         final SsoManager ssoManager,
@@ -104,31 +122,21 @@ public class SsoCallbackLogic implements CallbackLogic<Result, PlayWebContext> {
         final SystemEntityClient entityClient,
         final AuthServiceClient authClient,
         final CookieConfigs cookieConfigs) {
-      _oidcCallbackLogic =
+      oidcCallbackLogic =
           new OidcCallbackLogic(
               ssoManager, systemOperationContext, entityClient, authClient, cookieConfigs);
     }
 
     @Override
-    public Result perform(
-        PlayWebContext context,
+    public Object perform(
         Config config,
-        HttpActionAdapter<Result, PlayWebContext> httpActionAdapter,
-        String defaultUrl,
-        Boolean saveInSession,
-        Boolean multiProfile,
-        Boolean renewSession,
-        String defaultClient) {
-      if (SsoProvider.SsoProtocol.OIDC.equals(_ssoManager.getSsoProvider().protocol())) {
-        return _oidcCallbackLogic.perform(
-            context,
-            config,
-            httpActionAdapter,
-            defaultUrl,
-            saveInSession,
-            multiProfile,
-            renewSession,
-            defaultClient);
+        String inputDefaultUrl,
+        Boolean inputRenewSession,
+        String defaultClient,
+        FrameworkParameters parameters) {
+      if (SsoProvider.SsoProtocol.OIDC.equals(ssoManager.getSsoProvider().protocol())) {
+        return oidcCallbackLogic.perform(
+            config, inputDefaultUrl, inputRenewSession, defaultClient, parameters);
       }
       // Should never occur.
       throw new UnsupportedOperationException(
@@ -137,18 +145,18 @@ public Result perform(
   }
 
   private boolean shouldHandleCallback(final String protocol) {
-    if (!_ssoManager.isSsoEnabled()) {
+    if (!ssoManager.isSsoEnabled()) {
       return false;
     }
     updateConfig();
-    return _ssoManager.getSsoProvider().protocol().getCommonName().equals(protocol);
+    return ssoManager.getSsoProvider().protocol().getCommonName().equals(protocol);
   }
 
   private void updateConfig() {
     final Clients clients = new Clients();
     final List<Client> clientList = new ArrayList<>();
-    clientList.add(_ssoManager.getSsoProvider().client());
+    clientList.add(ssoManager.getSsoProvider().client());
     clients.setClients(clientList);
-    _config.setClients(clients);
+    config.setClients(clients);
   }
 }
diff --git a/datahub-frontend/app/controllers/TrackingController.java b/datahub-frontend/app/controllers/TrackingController.java
index 254a8cc640d0c..5d12c96ed77cb 100644
--- a/datahub-frontend/app/controllers/TrackingController.java
+++ b/datahub-frontend/app/controllers/TrackingController.java
@@ -22,23 +22,23 @@
 @Singleton
 public class TrackingController extends Controller {
 
-  private final Logger _logger = LoggerFactory.getLogger(TrackingController.class.getName());
+  private static final Logger logger = LoggerFactory.getLogger(TrackingController.class.getName());
 
-  private final String _topic;
+  private final String topic;
 
-  @Inject KafkaTrackingProducer _producer;
+  @Inject KafkaTrackingProducer producer;
 
-  @Inject AuthServiceClient _authClient;
+  @Inject AuthServiceClient authClient;
 
   @Inject
   public TrackingController(@Nonnull Config config) {
-    _topic = config.getString("analytics.tracking.topic");
+    topic = config.getString("analytics.tracking.topic");
   }
 
   @Security.Authenticated(Authenticator.class)
   @Nonnull
   public Result track(Http.Request request) throws Exception {
-    if (!_producer.isEnabled()) {
+    if (!producer.isEnabled()) {
       // If tracking is disabled, simply return a 200.
       return status(200);
     }
@@ -51,15 +51,15 @@ public Result track(Http.Request request) throws Exception {
     }
     final String actor = request.session().data().get(ACTOR);
     try {
-      _logger.debug(
+      logger.debug(
           String.format("Emitting product analytics event. actor: %s, event: %s", actor, event));
       final ProducerRecord<String, String> record =
-          new ProducerRecord<>(_topic, actor, event.toString());
-      _producer.send(record);
-      _authClient.track(event.toString());
+          new ProducerRecord<>(topic, actor, event.toString());
+      producer.send(record);
+      authClient.track(event.toString());
       return ok();
     } catch (Exception e) {
-      _logger.error(
+      logger.error(
           String.format(
               "Failed to emit product analytics event. actor: %s, event: %s", actor, event));
       return internalServerError(e.getMessage());
diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle
index ab4ce405a5541..5cc5af50d217b 100644
--- a/datahub-frontend/build.gradle
+++ b/datahub-frontend/build.gradle
@@ -4,14 +4,28 @@ plugins {
   id 'org.gradle.playframework'
 }
 
-apply from: "../gradle/versioning/versioning.gradle"
+apply from: '../gradle/versioning/versioning.gradle'
 apply from: './play.gradle'
+apply from: '../gradle/coverage/java-coverage.gradle'
 
 ext {
   docker_repo = 'datahub-frontend-react'
   docker_dir = 'datahub-frontend'
 }
 
+java {
+  toolchain {
+    languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+  }
+}
+
+test {
+  jacoco {
+    // jacoco instrumentation is failing when dealing with code of this dependency, excluding it.
+    excludes = ["com/gargoylesoftware/**"]
+  }
+}
+
 model {
   // Must specify the dependency here as "stage" is added by rule based model.
   tasks.myTar {
diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf
index be57a33b13564..db982b595e248 100644
--- a/datahub-frontend/conf/application.conf
+++ b/datahub-frontend/conf/application.conf
@@ -184,6 +184,7 @@ auth.oidc.responseMode = ${?AUTH_OIDC_RESPONSE_MODE}
 auth.oidc.useNonce = ${?AUTH_OIDC_USE_NONCE}
 auth.oidc.customParam.resource = ${?AUTH_OIDC_CUSTOM_PARAM_RESOURCE}
 auth.oidc.readTimeout = ${?AUTH_OIDC_READ_TIMEOUT}
+auth.oidc.connectTimeout = ${?AUTH_OIDC_CONNECT_TIMEOUT}
 auth.oidc.extractJwtAccessTokenClaims = ${?AUTH_OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS} # Whether to extract claims from JWT access token.  Defaults to false.
 auth.oidc.preferredJwsAlgorithm = ${?AUTH_OIDC_PREFERRED_JWS_ALGORITHM} # Which jws algorithm to use
 auth.oidc.acrValues = ${?AUTH_OIDC_ACR_VALUES}
diff --git a/datahub-frontend/conf/logback.xml b/datahub-frontend/conf/logback.xml
index 78da231b4a71c..de37c56cba38a 100644
--- a/datahub-frontend/conf/logback.xml
+++ b/datahub-frontend/conf/logback.xml
@@ -61,7 +61,7 @@
   <logger name="com.linkedin" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
-  <logger name="controller" level="DEBUG">
+  <logger name="controllers" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
   <logger name="auth" level="DEBUG">
diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle
index ff43e4a93a80f..d513c3c232d9a 100644
--- a/datahub-frontend/play.gradle
+++ b/datahub-frontend/play.gradle
@@ -11,16 +11,29 @@ configurations {
   play
 }
 
+ext {
+  nimbusJoseJwtVersion = "9.41.2"
+  oauth2OidcSdkVersion = "11.20.1"
+}
+
 dependencies {
   implementation project(':datahub-web-react')
 
   constraints {
+    play(externalDependency.pac4j)
+    play(externalDependency.playPac4j)
+    play("com.nimbusds:oauth2-oidc-sdk:$oauth2OidcSdkVersion")
+    play("com.nimbusds:nimbus-jose-jwt:$nimbusJoseJwtVersion")
+    implementation(externalDependency.pac4j)
+    implementation(externalDependency.playPac4j)
+    implementation("com.nimbusds:nimbus-jose-jwt:$nimbusJoseJwtVersion")
+    testImplementation("com.nimbusds:oauth2-oidc-sdk:$oauth2OidcSdkVersion")
+
     play(externalDependency.jacksonDataBind)
-    play('com.nimbusds:oauth2-oidc-sdk:8.36.2')
-    play('com.nimbusds:nimbus-jose-jwt:8.18')
-    play('com.typesafe.akka:akka-actor_2.12:2.6.20')
+    play("com.typesafe.akka:akka-actor_$playScalaVersion:2.6.20")
     play(externalDependency.jsonSmart)
     play('io.netty:netty-all:4.1.114.Final')
+
     implementation(externalDependency.commonsText) {
       because("previous versions are vulnerable to CVE-2022-42889")
     }
@@ -42,18 +55,22 @@ dependencies {
   implementation externalDependency.antlr4Runtime
   implementation externalDependency.antlr4
   implementation externalDependency.akkaHttp
+  implementation externalDependency.akkaActor
+  implementation externalDependency.akkaStream
+  implementation externalDependency.akkaActorTyped
+  implementation externalDependency.akkaSlf4j
+  implementation externalDependency.akkaJackson
+  implementation externalDependency.akkaParsing
+  implementation externalDependency.akkaProtobuf
 
   implementation externalDependency.jerseyCore
   implementation externalDependency.jerseyGuava
 
-  implementation(externalDependency.pac4j) {
-    exclude group: "net.minidev", module: "json-smart"
-    exclude group: "com.nimbusds", module: "nimbus-jose-jwt"
-  }
-
-  implementation 'com.nimbusds:nimbus-jose-jwt:8.18'
-  implementation externalDependency.jsonSmart
+  implementation externalDependency.pac4j
   implementation externalDependency.playPac4j
+  implementation "com.nimbusds:nimbus-jose-jwt:$nimbusJoseJwtVersion"
+  implementation externalDependency.jsonSmart
+  
   implementation externalDependency.shiroCore
 
   implementation externalDependency.playCache
@@ -69,7 +86,7 @@ dependencies {
   testImplementation externalDependency.mockito
   testImplementation externalDependency.playTest
   testImplementation 'org.awaitility:awaitility:4.2.0'
-  testImplementation 'no.nav.security:mock-oauth2-server:0.3.1'
+  testImplementation 'no.nav.security:mock-oauth2-server:2.1.9'
   testImplementation 'org.junit-pioneer:junit-pioneer:1.9.1'
   testImplementation externalDependency.junitJupiterApi
   testRuntimeOnly externalDependency.junitJupiterEngine
@@ -78,7 +95,7 @@ dependencies {
   compileOnly externalDependency.lombok
   runtimeOnly externalDependency.guicePlay
   runtimeOnly (externalDependency.playDocs) {
-    exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
+    exclude group: 'com.typesafe.akka', module: "akka-http-core_$playScalaVersion"
   }
   runtimeOnly externalDependency.playGuice
   implementation externalDependency.log4j2Api
@@ -90,9 +107,9 @@ dependencies {
 
 play {
   platform {
-    playVersion = '2.8.21'
-    scalaVersion = '2.12'
-    javaVersion = JavaVersion.VERSION_11
+    playVersion = "2.8.22" // see also top level build.gradle
+    scalaVersion = "2.13"
+    javaVersion = JavaVersion.VERSION_17
   }
 
   injectedRoutesGenerator = true
diff --git a/datahub-frontend/test/app/ApplicationTest.java b/datahub-frontend/test/app/ApplicationTest.java
index 534cffb5cc7fe..3ad9e22857168 100644
--- a/datahub-frontend/test/app/ApplicationTest.java
+++ b/datahub-frontend/test/app/ApplicationTest.java
@@ -12,17 +12,27 @@
 import com.nimbusds.jwt.JWTParser;
 import controllers.routes;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.InetAddress;
+import java.net.URL;
 import java.text.ParseException;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import no.nav.security.mock.oauth2.MockOAuth2Server;
+import no.nav.security.mock.oauth2.http.OAuth2HttpRequest;
+import no.nav.security.mock.oauth2.http.OAuth2HttpResponse;
+import no.nav.security.mock.oauth2.http.Route;
 import no.nav.security.mock.oauth2.token.DefaultOAuth2TokenCallback;
+import okhttp3.Headers;
 import okhttp3.mockwebserver.MockResponse;
 import okhttp3.mockwebserver.MockWebServer;
 import org.awaitility.Awaitility;
 import org.awaitility.Durations;
+import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -30,6 +40,8 @@
 import org.junitpioneer.jupiter.SetEnvironmentVariable;
 import org.openqa.selenium.Cookie;
 import org.openqa.selenium.htmlunit.HtmlUnitDriver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import play.Application;
 import play.Environment;
 import play.Mode;
@@ -48,7 +60,9 @@
 @SetEnvironmentVariable(key = "AUTH_OIDC_JIT_PROVISIONING_ENABLED", value = "false")
 @SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_ID", value = "testclient")
 @SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_SECRET", value = "testsecret")
+@SetEnvironmentVariable(key = "AUTH_VERBOSE_LOGGING", value = "true")
 public class ApplicationTest extends WithBrowser {
+  private static final Logger logger = LoggerFactory.getLogger(ApplicationTest.class);
   private static final String ISSUER_ID = "testIssuer";
 
   @Override
@@ -80,41 +94,34 @@ public int gmsServerPort() {
     return providePort() + 2;
   }
 
-  private MockOAuth2Server _oauthServer;
-  private MockWebServer _gmsServer;
+  private MockOAuth2Server oauthServer;
+  private Thread oauthServerThread;
+  private CompletableFuture<Void> oauthServerStarted;
 
-  private String _wellKnownUrl;
+  private MockWebServer gmsServer;
+
+  private String wellKnownUrl;
 
   private static final String TEST_USER = "urn:li:corpuser:testUser@myCompany.com";
   private static final String TEST_TOKEN = "faketoken_YCpYIrjQH4sD3_rAc3VPPFg4";
 
   @BeforeAll
   public void init() throws IOException {
-    _gmsServer = new MockWebServer();
-    _gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested
-    _gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested
-    _gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested
-    _gmsServer.enqueue(new MockResponse().setBody(String.format("{\"value\":\"%s\"}", TEST_USER)));
-    _gmsServer.enqueue(
+    // Start Mock GMS
+    gmsServer = new MockWebServer();
+    gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested
+    gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested
+    gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested
+    gmsServer.enqueue(new MockResponse().setBody(String.format("{\"value\":\"%s\"}", TEST_USER)));
+    gmsServer.enqueue(
         new MockResponse().setBody(String.format("{\"accessToken\":\"%s\"}", TEST_TOKEN)));
-    _gmsServer.start(gmsServerPort());
-
-    _oauthServer = new MockOAuth2Server();
-    _oauthServer.enqueueCallback(
-        new DefaultOAuth2TokenCallback(
-            ISSUER_ID,
-            "testUser",
-            List.of(),
-            Map.of(
-                "email", "testUser@myCompany.com",
-                "groups", "myGroup"),
-            600));
-    _oauthServer.start(InetAddress.getByName("localhost"), oauthServerPort());
-
-    // Discovery url to authorization server metadata
-    _wellKnownUrl = _oauthServer.wellKnownUrl(ISSUER_ID).toString();
+    gmsServer.start(gmsServerPort());
 
+    // Start Mock Identity Provider
+    startMockOauthServer();
+    // Start Play Frontend
     startServer();
+    // Start Browser
     createBrowser();
 
     Awaitility.await().timeout(Durations.TEN_SECONDS).until(() -> app != null);
@@ -122,13 +129,131 @@ public void init() throws IOException {
 
   @AfterAll
   public void shutdown() throws IOException {
-    if (_gmsServer != null) {
-      _gmsServer.shutdown();
-    }
-    if (_oauthServer != null) {
-      _oauthServer.shutdown();
+    if (gmsServer != null) {
+      logger.info("Shutdown Mock GMS");
+      gmsServer.shutdown();
     }
+    logger.info("Shutdown Play Frontend");
     stopServer();
+    if (oauthServer != null) {
+      logger.info("Shutdown MockOAuth2Server");
+      oauthServer.shutdown();
+    }
+    if (oauthServerThread != null && oauthServerThread.isAlive()) {
+      logger.info("Shutdown MockOAuth2Server thread");
+      oauthServerThread.interrupt();
+      try {
+        oauthServerThread.join(2000); // Wait up to 2 seconds for thread to finish
+      } catch (InterruptedException e) {
+        logger.warn("Shutdown MockOAuth2Server thread failed to join.");
+      }
+    }
+  }
+
+  private void startMockOauthServer() {
+    // Configure HEAD responses
+    Route[] routes =
+        new Route[] {
+          new Route() {
+            @Override
+            public boolean match(@NotNull OAuth2HttpRequest oAuth2HttpRequest) {
+              return "HEAD".equals(oAuth2HttpRequest.getMethod())
+                  && (String.format("/%s/.well-known/openid-configuration", ISSUER_ID)
+                          .equals(oAuth2HttpRequest.getUrl().url().getPath())
+                      || String.format("/%s/token", ISSUER_ID)
+                          .equals(oAuth2HttpRequest.getUrl().url().getPath()));
+            }
+
+            @Override
+            public OAuth2HttpResponse invoke(OAuth2HttpRequest oAuth2HttpRequest) {
+              return new OAuth2HttpResponse(
+                  Headers.of(
+                      Map.of(
+                          "Content-Type", "application/json",
+                          "Cache-Control", "no-store",
+                          "Pragma", "no-cache",
+                          "Content-Length", "-1")),
+                  200,
+                  null,
+                  null);
+            }
+          }
+        };
+    oauthServer = new MockOAuth2Server(routes);
+    oauthServerStarted = new CompletableFuture<>();
+
+    // Create and start server in separate thread
+    oauthServerThread =
+        new Thread(
+            () -> {
+              try {
+                // Configure mock responses
+                oauthServer.enqueueCallback(
+                    new DefaultOAuth2TokenCallback(
+                        ISSUER_ID,
+                        "testUser",
+                        "JWT",
+                        List.of(),
+                        Map.of(
+                            "email", "testUser@myCompany.com",
+                            "groups", "myGroup"),
+                        600));
+
+                oauthServer.start(InetAddress.getByName("localhost"), oauthServerPort());
+
+                oauthServerStarted.complete(null);
+
+                // Keep thread alive until server is stopped
+                while (!Thread.currentThread().isInterrupted() && testServer.isRunning()) {
+                  try {
+                    Thread.sleep(1000);
+                  } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    break;
+                  }
+                }
+              } catch (Exception e) {
+                oauthServerStarted.completeExceptionally(e);
+              }
+            });
+
+    oauthServerThread.setDaemon(true); // Ensure thread doesn't prevent JVM shutdown
+    oauthServerThread.start();
+
+    // Wait for server to start with timeout
+    oauthServerStarted
+        .orTimeout(10, TimeUnit.SECONDS)
+        .whenComplete(
+            (result, throwable) -> {
+              if (throwable != null) {
+                if (throwable instanceof TimeoutException) {
+                  throw new RuntimeException(
+                      "MockOAuth2Server failed to start within timeout", throwable);
+                }
+                throw new RuntimeException("MockOAuth2Server failed to start", throwable);
+              }
+            });
+
+    // Discovery url to authorization server metadata
+    wellKnownUrl = oauthServer.wellKnownUrl(ISSUER_ID).toString();
+
+    // Wait for server to return configuration
+    // Validate mock server returns data
+    try {
+      URL url = new URL(wellKnownUrl);
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.setRequestMethod("GET");
+      int responseCode = conn.getResponseCode();
+      logger.info("Well-known endpoint response code: {}", responseCode);
+
+      if (responseCode != 200) {
+        throw new RuntimeException(
+            "MockOAuth2Server not accessible. Response code: " + responseCode);
+      }
+      logger.info("Successfully started MockOAuth2Server.");
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to connect to MockOAuth2Server", e);
+    }
   }
 
   @Test
@@ -158,7 +283,7 @@ public void testIndexNotFound() {
   public void testOpenIdConfig() {
     assertEquals(
         "http://localhost:" + oauthServerPort() + "/testIssuer/.well-known/openid-configuration",
-        _wellKnownUrl);
+        wellKnownUrl);
   }
 
   @Test
@@ -188,10 +313,10 @@ public void testHappyPathOidc() throws ParseException {
   @Test
   public void testAPI() throws ParseException {
     testHappyPathOidc();
-    int requestCount = _gmsServer.getRequestCount();
+    int requestCount = gmsServer.getRequestCount();
 
     browser.goTo("/api/v2/graphql/");
-    assertEquals(++requestCount, _gmsServer.getRequestCount());
+    assertEquals(++requestCount, gmsServer.getRequestCount());
   }
 
   @Test
@@ -201,8 +326,9 @@ public void testOidcRedirectToRequestedUrl() {
   }
 
   /**
-   * The Redirect Uri parameter is used to store a previous relative location within the app to be able to
-   * take a user back to their expected page. Redirecting to other domains should be blocked.
+   * The Redirect Uri parameter is used to store a previous relative location within the app to be
+   * able to take a user back to their expected page. Redirecting to other domains should be
+   * blocked.
    */
   @Test
   public void testInvalidRedirectUrl() {
diff --git a/datahub-frontend/test/oidc/OidcCallbackLogicTest.java b/datahub-frontend/test/oidc/OidcCallbackLogicTest.java
index f4784c29e91f2..9eb3833cbc897 100644
--- a/datahub-frontend/test/oidc/OidcCallbackLogicTest.java
+++ b/datahub-frontend/test/oidc/OidcCallbackLogicTest.java
@@ -1,64 +1,65 @@
 package oidc;
 
-import auth.sso.oidc.OidcConfigs;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
 import static auth.sso.oidc.OidcCallbackLogic.getGroupNames;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import java.util.Arrays;
+import java.util.Collection;
 import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
 import org.pac4j.core.profile.CommonProfile;
 
 public class OidcCallbackLogicTest {
 
-    @Test
-    public void testGetGroupsClaimNamesJsonArray() {
-        CommonProfile profile = createMockProfileWithAttribute("[\"group1\", \"group2\"]", "groupsClaimName");
-        Collection<String> result = getGroupNames(profile, "[\"group1\", \"group2\"]", "groupsClaimName");
-        assertEquals(Arrays.asList("group1", "group2"), result);
-    }
-    @Test
-    public void testGetGroupNamesWithSingleGroup() {
-        CommonProfile profile = createMockProfileWithAttribute("group1", "groupsClaimName");
-        Collection<String> result = getGroupNames(profile, "group1", "groupsClaimName");
-        assertEquals(Arrays.asList("group1"), result);
-    }
+  @Test
+  public void testGetGroupsClaimNamesJsonArray() {
+    CommonProfile profile =
+        createMockProfileWithAttribute("[\"group1\", \"group2\"]", "groupsClaimName");
+    Collection<String> result =
+        getGroupNames(profile, "[\"group1\", \"group2\"]", "groupsClaimName");
+    assertEquals(Arrays.asList("group1", "group2"), result);
+  }
 
-    @Test
-    public void testGetGroupNamesWithCommaSeparated() {
-        CommonProfile profile = createMockProfileWithAttribute("group1,group2", "groupsClaimName");
-        Collection<String> result = getGroupNames(profile, "group1,group2", "groupsClaimName");
-        assertEquals(Arrays.asList("group1", "group2"), result);
-    }
+  @Test
+  public void testGetGroupNamesWithSingleGroup() {
+    CommonProfile profile = createMockProfileWithAttribute("group1", "groupsClaimName");
+    Collection<String> result = getGroupNames(profile, "group1", "groupsClaimName");
+    assertEquals(Arrays.asList("group1"), result);
+  }
 
-    @Test
-    public void testGetGroupNamesWithCollection() {
-        CommonProfile profile = createMockProfileWithAttribute(Arrays.asList("group1", "group2"), "groupsClaimName");
-        Collection<String> result = getGroupNames(profile, Arrays.asList("group1", "group2"), "groupsClaimName");
-        assertEquals(Arrays.asList("group1", "group2"), result);
-    }
-    // Helper method to create a mock CommonProfile with given attribute
-    private CommonProfile createMockProfileWithAttribute(Object attribute, String attributeName) {
-        CommonProfile profile = mock(CommonProfile.class);
+  @Test
+  public void testGetGroupNamesWithCommaSeparated() {
+    CommonProfile profile = createMockProfileWithAttribute("group1,group2", "groupsClaimName");
+    Collection<String> result = getGroupNames(profile, "group1,group2", "groupsClaimName");
+    assertEquals(Arrays.asList("group1", "group2"), result);
+  }
+
+  @Test
+  public void testGetGroupNamesWithCollection() {
+    CommonProfile profile =
+        createMockProfileWithAttribute(Arrays.asList("group1", "group2"), "groupsClaimName");
+    Collection<String> result =
+        getGroupNames(profile, Arrays.asList("group1", "group2"), "groupsClaimName");
+    assertEquals(Arrays.asList("group1", "group2"), result);
+  }
 
-        // Mock for getAttribute(String)
-        when(profile.getAttribute(attributeName)).thenReturn(attribute);
+  // Helper method to create a mock CommonProfile with given attribute
+  private CommonProfile createMockProfileWithAttribute(Object attribute, String attributeName) {
+    CommonProfile profile = mock(CommonProfile.class);
 
-        // Mock for getAttribute(String, Class<T>)
-        if (attribute instanceof Collection) {
-            when(profile.getAttribute(attributeName, Collection.class)).thenReturn((Collection) attribute);
-        } else if (attribute instanceof String) {
-            when(profile.getAttribute(attributeName, String.class)).thenReturn((String) attribute);
-        }
-        // Add more conditions here if needed for other types
+    // Mock for getAttribute(String)
+    when(profile.getAttribute(attributeName)).thenReturn(attribute);
 
-        return profile;
+    // Mock for getAttribute(String, Class<T>)
+    if (attribute instanceof Collection) {
+      when(profile.getAttribute(attributeName, Collection.class))
+          .thenReturn((Collection) attribute);
+    } else if (attribute instanceof String) {
+      when(profile.getAttribute(attributeName, String.class)).thenReturn((String) attribute);
     }
+    // Add more conditions here if needed for other types
+
+    return profile;
+  }
 }
diff --git a/datahub-frontend/test/security/OidcConfigurationTest.java b/datahub-frontend/test/security/OidcConfigurationTest.java
index 1c52d45af5f9e..ec19979c56120 100644
--- a/datahub-frontend/test/security/OidcConfigurationTest.java
+++ b/datahub-frontend/test/security/OidcConfigurationTest.java
@@ -23,9 +23,9 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
 import org.pac4j.oidc.client.OidcClient;
-import org.json.JSONObject;
 
 public class OidcConfigurationTest {
 
@@ -328,17 +328,28 @@ public void readPreferredJwsAlgorithmPropagationFromConfig() {
     oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR);
     OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder);
     OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
-    assertEquals("RS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString());
+    assertEquals(
+        "RS256",
+        ((OidcClient) oidcProvider.client())
+            .getConfiguration()
+            .getPreferredJwsAlgorithm()
+            .toString());
   }
 
   @Test
   public void readPreferredJwsAlgorithmPropagationFromJSON() {
-    final String SSO_SETTINGS_JSON_STR = new JSONObject().put(PREFERRED_JWS_ALGORITHM, "HS256").toString();
+    final String SSO_SETTINGS_JSON_STR =
+        new JSONObject().put(PREFERRED_JWS_ALGORITHM, "HS256").toString();
     CONFIG.withValue(OIDC_PREFERRED_JWS_ALGORITHM, ConfigValueFactory.fromAnyRef("RS256"));
     OidcConfigs.Builder oidcConfigsBuilder = new OidcConfigs.Builder();
     oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR);
     OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder);
     OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
-    assertEquals("HS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString());
+    assertEquals(
+        "HS256",
+        ((OidcClient) oidcProvider.client())
+            .getConfiguration()
+            .getPreferredJwsAlgorithm()
+            .toString());
   }
 }
diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle
index 49a7fa7fbfbc2..47ada8e9929dd 100644
--- a/datahub-graphql-core/build.gradle
+++ b/datahub-graphql-core/build.gradle
@@ -3,6 +3,7 @@ plugins {
     id "io.github.kobylynskyi.graphql.codegen" version "4.1.1"
 }
 
+apply from: '../gradle/coverage/java-coverage.gradle'
 
 dependencies {
     implementation project(':metadata-service:restli-client-api')
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
index 5b265b6714452..94f0e8a055b70 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -63,9 +63,11 @@
 import com.linkedin.datahub.graphql.generated.Domain;
 import com.linkedin.datahub.graphql.generated.ERModelRelationship;
 import com.linkedin.datahub.graphql.generated.ERModelRelationshipProperties;
+import com.linkedin.datahub.graphql.generated.Entity;
 import com.linkedin.datahub.graphql.generated.EntityPath;
 import com.linkedin.datahub.graphql.generated.EntityRelationship;
 import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
+import com.linkedin.datahub.graphql.generated.FacetMetadata;
 import com.linkedin.datahub.graphql.generated.ForeignKeyConstraint;
 import com.linkedin.datahub.graphql.generated.FormActorAssignment;
 import com.linkedin.datahub.graphql.generated.FreshnessContract;
@@ -312,6 +314,7 @@
 import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.type.PropertyValueResolver;
+import com.linkedin.datahub.graphql.resolvers.type.ResolvedActorResolver;
 import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserResetTokenResolver;
@@ -1315,7 +1318,8 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
               .dataFetcher("updateQuery", new UpdateQueryResolver(this.queryService))
               .dataFetcher("deleteQuery", new DeleteQueryResolver(this.queryService))
               .dataFetcher(
-                  "createDataProduct", new CreateDataProductResolver(this.dataProductService))
+                  "createDataProduct",
+                  new CreateDataProductResolver(this.dataProductService, this.entityService))
               .dataFetcher(
                   "updateDataProduct", new UpdateDataProductResolver(this.dataProductService))
               .dataFetcher(
@@ -1472,6 +1476,19 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder
                     "entity",
                     new EntityTypeResolver(
                         entityTypes, (env) -> ((BrowsePathEntry) env.getSource()).getEntity())))
+        .type(
+            "FacetMetadata",
+            typeWiring ->
+                typeWiring.dataFetcher(
+                    "entity",
+                    new EntityTypeResolver(
+                        entityTypes,
+                        (env) -> {
+                          FacetMetadata facetMetadata = env.getSource();
+                          return facetMetadata.getEntity() != null
+                              ? facetMetadata.getEntity()
+                              : null;
+                        })))
         .type(
             "LineageRelationship",
             typeWiring ->
@@ -1730,12 +1747,22 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
         .type(
             "InstitutionalMemoryMetadata",
             typeWiring ->
-                typeWiring.dataFetcher(
-                    "author",
-                    new LoadableTypeResolver<>(
-                        corpUserType,
-                        (env) ->
-                            ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())))
+                typeWiring
+                    .dataFetcher(
+                        "author",
+                        new LoadableTypeResolver<>(
+                            corpUserType,
+                            (env) ->
+                                ((InstitutionalMemoryMetadata) env.getSource())
+                                    .getAuthor()
+                                    .getUrn()))
+                    .dataFetcher(
+                        "actor",
+                        new EntityTypeResolver(
+                            this.entityTypes,
+                            (env) ->
+                                (Entity)
+                                    ((InstitutionalMemoryMetadata) env.getSource()).getActor())))
         .type(
             "DatasetStatsSummary",
             typeWiring ->
@@ -2242,6 +2269,7 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) {
             "HyperParameterValueType",
             typeWiring -> typeWiring.typeResolver(new HyperParameterValueTypeResolver()))
         .type("PropertyValue", typeWiring -> typeWiring.typeResolver(new PropertyValueResolver()))
+        .type("ResolvedActor", typeWiring -> typeWiring.typeResolver(new ResolvedActorResolver()))
         .type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver()))
         .type(
             "TimeSeriesAspect",
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java
index 0fe6e5de0cac6..d9b8008d46286 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java
@@ -84,8 +84,21 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart(
       final DateTime end,
       final String title,
       final DateInterval interval) {
-    final DateRange dateRange =
-        new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis()));
+
+    final DateRange dateRange;
+
+    // adjust month to show 1st of month rather than last day of previous month
+    if (interval == DateInterval.MONTH) {
+      dateRange =
+          new DateRange(
+              String.valueOf(beginning.plusDays(1).getMillis()), // Shift start by 1 day
+              String.valueOf(end.plusDays(1).getMillis()) // Shift end by 1 day
+              );
+    } else {
+      // week display starting Sundays
+      dateRange =
+          new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis()));
+    }
 
     final List<NamedLine> timeSeriesLines =
         _analyticsService.getTimeseriesChart(
@@ -96,6 +109,7 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart(
             ImmutableMap.of(),
             Collections.emptyMap(),
             Optional.of("browserId"));
+
     return TimeSeriesChart.builder()
         .setTitle(title)
         .setDateRange(dateRange)
@@ -111,7 +125,7 @@ private AnalyticsChart getTopUsersChart(OperationContext opContext) {
       final DateRange trailingMonthDateRange = dateUtil.getTrailingMonthDateRange();
       final List<String> columns = ImmutableList.of("Name", "Title", "Email");
 
-      final String topUsersTitle = "Top Users";
+      final String topUsersTitle = "Top Users (Last 30 Days)";
       final List<Row> topUserRows =
           _analyticsService.getTopNTableChart(
               _analyticsService.getUsageIndexName(),
@@ -184,7 +198,7 @@ private Row buildNewUsersRow(@Nonnull final SearchEntity entity) {
   private AnalyticsChart getNewUsersChart(OperationContext opContext) {
     try {
       final List<String> columns = ImmutableList.of("Name", "Title", "Email");
-      final String newUsersTitle = "New Users";
+      final String newUsersTitle = "Active Users (Last 30 Days)";
       final SearchResult result = searchForNewUsers(opContext);
       final List<Row> newUserRows = new ArrayList<>();
       for (SearchEntity entity : result.getEntities()) {
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java
index a17745948eb82..88ac29b72dee8 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java
@@ -1,19 +1,14 @@
 package com.linkedin.datahub.graphql.analytics.service;
 
+import static com.linkedin.metadata.Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.CORP_USER_INFO_ASPECT_NAME;
 
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.dashboard.DashboardInfo;
-import com.linkedin.datahub.graphql.generated.BarSegment;
-import com.linkedin.datahub.graphql.generated.Cell;
-import com.linkedin.datahub.graphql.generated.Entity;
-import com.linkedin.datahub.graphql.generated.EntityProfileParams;
-import com.linkedin.datahub.graphql.generated.LinkParams;
-import com.linkedin.datahub.graphql.generated.NamedBar;
-import com.linkedin.datahub.graphql.generated.Row;
-import com.linkedin.datahub.graphql.generated.SearchParams;
+import com.linkedin.datahub.graphql.generated.*;
 import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
 import com.linkedin.dataplatform.DataPlatformInfo;
 import com.linkedin.dataset.DatasetProperties;
@@ -22,6 +17,7 @@
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.glossary.GlossaryTermInfo;
+import com.linkedin.identity.CorpUserEditableInfo;
 import com.linkedin.identity.CorpUserInfo;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.key.GlossaryTermKey;
@@ -35,6 +31,7 @@
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.extern.slf4j.Slf4j;
@@ -169,36 +166,79 @@ public static void convertToUserInfoRows(
     final Map<Urn, EntityResponse> gmsResponseByUser =
         entityClient.batchGetV2(
             opContext,
-            CORP_USER_INFO_ASPECT_NAME,
+            CORP_USER_ENTITY_NAME,
             userUrns,
-            ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME));
-    final Map<Urn, CorpUserInfo> urnToCorpUserInfo =
+            ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME, CORP_USER_EDITABLE_INFO_ASPECT_NAME));
+    final Stream<Map.Entry<Urn, EntityResponse>> entityStream =
         gmsResponseByUser.entrySet().stream()
             .filter(
                 entry ->
                     entry.getValue() != null
-                        && entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME))
-            .collect(
-                Collectors.toMap(
-                    Map.Entry::getKey,
-                    entry ->
+                        && (entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME)
+                            || entry
+                                .getValue()
+                                .getAspects()
+                                .containsKey(CORP_USER_EDITABLE_INFO_ASPECT_NAME)));
+    final Map<Urn, Pair<CorpUserInfo, CorpUserEditableInfo>> urnToCorpUserInfo =
+        entityStream.collect(
+            Collectors.toMap(
+                Map.Entry::getKey,
+                entry -> {
+                  CorpUserInfo userInfo = null;
+                  CorpUserEditableInfo editableInfo = null;
+                  try {
+                    userInfo =
                         new CorpUserInfo(
                             entry
                                 .getValue()
                                 .getAspects()
                                 .get(CORP_USER_INFO_ASPECT_NAME)
                                 .getValue()
-                                .data())));
+                                .data());
+                  } catch (Exception e) {
+                    // nothing to do
+                  }
+                  try {
+
+                    editableInfo =
+                        new CorpUserEditableInfo(
+                            entry
+                                .getValue()
+                                .getAspects()
+                                .get(CORP_USER_EDITABLE_INFO_ASPECT_NAME)
+                                .getValue()
+                                .data());
+                  } catch (Exception e) {
+                    // nothing to do
+                  }
+
+                  return Pair.of(userInfo, editableInfo);
+                }));
     // Populate a row with the user link, title, and email.
     rows.forEach(
         row -> {
           Urn urn = UrnUtils.getUrn(row.getCells().get(0).getValue());
           EntityResponse response = gmsResponseByUser.get(urn);
           String maybeDisplayName = response != null ? getUserName(response).orElse(null) : null;
-          String maybeEmail =
-              urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getEmail() : null;
-          String maybeTitle =
-              urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getTitle() : null;
+          String maybeEmail = null;
+          String maybeTitle = null;
+          if (urnToCorpUserInfo.containsKey(urn)) {
+            Pair<CorpUserInfo, CorpUserEditableInfo> pair = urnToCorpUserInfo.get(urn);
+            if (pair.getLeft() != null) {
+              CorpUserInfo userInfo = pair.getLeft();
+              maybeEmail = userInfo.getEmail();
+              maybeTitle = userInfo.getTitle();
+            }
+            if (pair.getRight() != null) {
+              CorpUserEditableInfo userInfo = pair.getRight();
+              if (maybeEmail == null) {
+                maybeEmail = userInfo.getEmail();
+              }
+              if (maybeTitle == null) {
+                maybeTitle = userInfo.getTitle();
+              }
+            }
+          }
           if (maybeDisplayName != null) {
             row.getCells().get(0).setValue(maybeDisplayName);
           }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java
index ca60acaa80538..c25d6af75fe76 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java
@@ -339,6 +339,11 @@ public static boolean canManageStructuredProperties(@Nonnull QueryContext contex
         context.getOperationContext(), PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE);
   }
 
+  public static boolean canViewStructuredPropertiesPage(@Nonnull QueryContext context) {
+    return AuthUtil.isAuthorized(
+        context.getOperationContext(), PoliciesConfig.VIEW_STRUCTURED_PROPERTIES_PAGE_PRIVILEGE);
+  }
+
   public static boolean canManageForms(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
         context.getOperationContext(), PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java
index b1101ae3ee865..8297392e642d5 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java
@@ -93,6 +93,10 @@ public CompletableFuture<AuthenticatedUser> get(DataFetchingEnvironment environm
                 BusinessAttributeAuthorizationUtils.canCreateBusinessAttribute(context));
             platformPrivileges.setManageBusinessAttributes(
                 BusinessAttributeAuthorizationUtils.canManageBusinessAttribute(context));
+            platformPrivileges.setManageStructuredProperties(
+                AuthorizationUtils.canManageStructuredProperties(context));
+            platformPrivileges.setViewStructuredPropertiesPage(
+                AuthorizationUtils.canViewStructuredPropertiesPage(context));
             // Construct and return authenticated user object.
             final AuthenticatedUser authUser = new AuthenticatedUser();
             authUser.setCorpUser(corpUser);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java
index 259d05c631557..3647eb55b2583 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java
@@ -188,6 +188,7 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
             .setDataContractsEnabled(_featureFlags.isDataContractsEnabled())
             .setEditableDatasetNameEnabled(_featureFlags.isEditableDatasetNameEnabled())
             .setShowSeparateSiblings(_featureFlags.isShowSeparateSiblings())
+            .setShowManageStructuredProperties(_featureFlags.isShowManageStructuredProperties())
             .build();
 
     appConfig.setFeatureFlags(featureFlagsConfig);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/CreateDataProductResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/CreateDataProductResolver.java
index 470267264f12f..8bee544ca55c3 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/CreateDataProductResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/CreateDataProductResolver.java
@@ -10,8 +10,11 @@
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.CreateDataProductInput;
 import com.linkedin.datahub.graphql.generated.DataProduct;
+import com.linkedin.datahub.graphql.generated.OwnerEntityType;
+import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils;
 import com.linkedin.datahub.graphql.types.dataproduct.mappers.DataProductMapper;
 import com.linkedin.entity.EntityResponse;
+import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.service.DataProductService;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
@@ -24,6 +27,7 @@
 public class CreateDataProductResolver implements DataFetcher<CompletableFuture<DataProduct>> {
 
   private final DataProductService _dataProductService;
+  private final EntityService _entityService;
 
   @Override
   public CompletableFuture<DataProduct> get(final DataFetchingEnvironment environment)
@@ -56,6 +60,8 @@ public CompletableFuture<DataProduct> get(final DataFetchingEnvironment environm
                 context.getOperationContext(),
                 dataProductUrn,
                 UrnUtils.getUrn(input.getDomainUrn()));
+            OwnerUtils.addCreatorAsOwner(
+                context, dataProductUrn.toString(), OwnerEntityType.CORP_USER, _entityService);
             EntityResponse response =
                 _dataProductService.getDataProductEntityResponse(
                     context.getOperationContext(), dataProductUrn);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java
index 8ead47aa65ceb..33b1555b73fab 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java
@@ -15,20 +15,22 @@
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.query.filter.SortOrder;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchResult;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
 
 /** Lists all ingestion sources stored within DataHub. Requires the MANAGE_INGESTION privilege. */
+@Slf4j
 public class ListIngestionSourcesResolver
     implements DataFetcher<CompletableFuture<ListIngestionSourcesResult>> {
 
@@ -57,6 +59,22 @@ public CompletableFuture<ListIngestionSourcesResult> get(
       final List<FacetFilterInput> filters =
           input.getFilters() == null ? Collections.emptyList() : input.getFilters();
 
+      // construct sort criteria, defaulting to systemCreated
+      final SortCriterion sortCriterion;
+
+      // if query is expecting to sort by something, use that
+      final com.linkedin.datahub.graphql.generated.SortCriterion sortCriterionInput =
+          input.getSort();
+      if (sortCriterionInput != null) {
+        sortCriterion =
+            new SortCriterion()
+                .setField(sortCriterionInput.getField())
+                .setOrder(SortOrder.valueOf(sortCriterionInput.getSortOrder().name()));
+      } else {
+        // TODO: default to last executed
+        sortCriterion = null;
+      }
+
       return GraphQLConcurrencyUtils.supplyAsync(
           () -> {
             try {
@@ -69,33 +87,24 @@ public CompletableFuture<ListIngestionSourcesResult> get(
                       Constants.INGESTION_SOURCE_ENTITY_NAME,
                       query,
                       buildFilter(filters, Collections.emptyList()),
-                      null,
+                      sortCriterion != null ? List.of(sortCriterion) : null,
                       start,
                       count);
 
+              final List<Urn> entitiesUrnList =
+                  gmsResult.getEntities().stream().map(SearchEntity::getEntity).toList();
               // Then, resolve all ingestion sources
               final Map<Urn, EntityResponse> entities =
                   _entityClient.batchGetV2(
                       context.getOperationContext(),
                       Constants.INGESTION_SOURCE_ENTITY_NAME,
-                      new HashSet<>(
-                          gmsResult.getEntities().stream()
-                              .map(SearchEntity::getEntity)
-                              .collect(Collectors.toList())),
+                      new HashSet<>(entitiesUrnList),
                       ImmutableSet.of(
                           Constants.INGESTION_INFO_ASPECT_NAME,
                           Constants.INGESTION_SOURCE_KEY_ASPECT_NAME));
 
-              final Collection<EntityResponse> sortedEntities =
-                  entities.values().stream()
-                      .sorted(
-                          Comparator.comparingLong(
-                              s ->
-                                  -s.getAspects()
-                                      .get(Constants.INGESTION_SOURCE_KEY_ASPECT_NAME)
-                                      .getCreated()
-                                      .getTime()))
-                      .collect(Collectors.toList());
+              final List<EntityResponse> entitiesOrdered =
+                  entitiesUrnList.stream().map(entities::get).filter(Objects::nonNull).toList();
 
               // Now that we have entities we can bind this to a result.
               final ListIngestionSourcesResult result = new ListIngestionSourcesResult();
@@ -103,7 +112,7 @@ public CompletableFuture<ListIngestionSourcesResult> get(
               result.setCount(gmsResult.getPageSize());
               result.setTotal(gmsResult.getNumEntities());
               result.setIngestionSources(
-                  IngestionResolverUtils.mapIngestionSources(sortedEntities));
+                  IngestionResolverUtils.mapIngestionSources(entitiesOrdered));
               return result;
 
             } catch (Exception e) {
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java
index 29b71d95ad974..31ed2de7a6d51 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java
@@ -66,11 +66,17 @@ public CompletableFuture<AggregateResults> get(DataFetchingEnvironment environme
 
           final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters());
 
-          final SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags());
+          final SearchFlags searchFlags =
+              input.getSearchFlags() != null
+                  ? mapInputFlags(context, input.getSearchFlags())
+                  : new SearchFlags();
 
           final List<String> facets =
               input.getFacets() != null && input.getFacets().size() > 0 ? input.getFacets() : null;
 
+          // do not include default facets if we're requesting any facets specifically
+          searchFlags.setIncludeDefaultFacets(facets == null || facets.size() <= 0);
+
           List<String> finalEntities =
               maybeResolvedView != null
                   ? SearchUtils.intersectEntityTypes(
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java
index d103704146d39..29bc3a82a1649 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java
@@ -2,19 +2,28 @@
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
 import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.*;
+import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.getEntityNames;
 
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
+import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput;
 import com.linkedin.datahub.graphql.generated.SearchResults;
 import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.query.SearchFlags;
+import com.linkedin.metadata.query.filter.Condition;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.search.SearchResult;
 import com.linkedin.metadata.service.ViewService;
+import com.linkedin.metadata.utils.CriterionUtils;
 import com.linkedin.view.DataHubViewInfo;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
@@ -64,24 +73,7 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
               ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters());
 
           SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags());
-          List<SortCriterion> sortCriteria;
-          if (input.getSortInput() != null) {
-            if (input.getSortInput().getSortCriteria() != null) {
-              sortCriteria =
-                  input.getSortInput().getSortCriteria().stream()
-                      .map(SearchUtils::mapSortCriterion)
-                      .collect(Collectors.toList());
-            } else {
-              sortCriteria =
-                  input.getSortInput().getSortCriterion() != null
-                      ? Collections.singletonList(
-                          mapSortCriterion(input.getSortInput().getSortCriterion()))
-                      : Collections.emptyList();
-            }
-
-          } else {
-            sortCriteria = Collections.emptyList();
-          }
+          List<SortCriterion> sortCriteria = SearchUtils.getSortCriteria(input.getSortInput());
 
           try {
             log.debug(
@@ -101,6 +93,14 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
               return SearchUtils.createEmptySearchResults(start, count);
             }
 
+            boolean shouldIncludeStructuredPropertyFacets =
+                input.getSearchFlags() != null
+                        && input.getSearchFlags().getIncludeStructuredPropertyFacets() != null
+                    ? input.getSearchFlags().getIncludeStructuredPropertyFacets()
+                    : false;
+            List<String> structuredPropertyFacets =
+                shouldIncludeStructuredPropertyFacets ? getStructuredPropertyFacets(context) : null;
+
             return UrnSearchResultsMapper.map(
                 context,
                 _entityClient.searchAcrossEntities(
@@ -113,7 +113,8 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
                         : baseFilter,
                     start,
                     count,
-                    sortCriteria));
+                    sortCriteria,
+                    structuredPropertyFacets));
           } catch (Exception e) {
             log.error(
                 "Failed to execute search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}",
@@ -133,4 +134,45 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
         this.getClass().getSimpleName(),
         "get");
   }
+
+  private List<String> getStructuredPropertyFacets(final QueryContext context) {
+    try {
+      SearchFlags searchFlags = new SearchFlags().setSkipCache(true);
+      SearchResult result =
+          _entityClient.searchAcrossEntities(
+              context.getOperationContext().withSearchFlags(flags -> searchFlags),
+              getEntityNames(ImmutableList.of(EntityType.STRUCTURED_PROPERTY)),
+              "*",
+              createStructuredPropertyFilter(),
+              0,
+              100,
+              Collections.emptyList(),
+              null);
+      return result.getEntities().stream()
+          .map(entity -> String.format("structuredProperties.%s", entity.getEntity().getId()))
+          .collect(Collectors.toList());
+    } catch (Exception e) {
+      log.error("Failed to get structured property facets to filter on", e);
+      return Collections.emptyList();
+    }
+  }
+
+  private Filter createStructuredPropertyFilter() {
+    return new Filter()
+        .setOr(
+            new ConjunctiveCriterionArray(
+                ImmutableList.of(
+                    new ConjunctiveCriterion()
+                        .setAnd(
+                            new CriterionArray(
+                                ImmutableList.of(
+                                    CriterionUtils.buildCriterion(
+                                        "filterStatus", Condition.EQUAL, "ENABLED")))),
+                    new ConjunctiveCriterion()
+                        .setAnd(
+                            new CriterionArray(
+                                ImmutableList.of(
+                                    CriterionUtils.buildCriterion(
+                                        "showInSearchFilters", Condition.EQUAL, "true")))))));
+  }
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java
index 04777c3fcdb4e..a01b3aaec9c98 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java
@@ -22,6 +22,7 @@
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
 import com.linkedin.datahub.graphql.generated.SearchResults;
+import com.linkedin.datahub.graphql.generated.SearchSortInput;
 import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper;
 import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
 import com.linkedin.metadata.query.SearchFlags;
@@ -326,4 +327,25 @@ public static SearchResults createEmptySearchResults(final int start, final int
     result.setFacets(new ArrayList<>());
     return result;
   }
+
+  public static List<SortCriterion> getSortCriteria(@Nullable final SearchSortInput sortInput) {
+    List<SortCriterion> sortCriteria;
+    if (sortInput != null) {
+      if (sortInput.getSortCriteria() != null) {
+        sortCriteria =
+            sortInput.getSortCriteria().stream()
+                .map(SearchUtils::mapSortCriterion)
+                .collect(Collectors.toList());
+      } else {
+        sortCriteria =
+            sortInput.getSortCriterion() != null
+                ? Collections.singletonList(mapSortCriterion(sortInput.getSortCriterion()))
+                : new ArrayList<>();
+      }
+    } else {
+      sortCriteria = new ArrayList<>();
+    }
+
+    return sortCriteria;
+  }
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java
index 328f63b893d06..7d232748f0d93 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java
@@ -1,7 +1,8 @@
 package com.linkedin.datahub.graphql.resolvers.structuredproperties;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
-import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
+import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithUrn;
+import static com.linkedin.metadata.Constants.*;
 
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.SetMode;
@@ -12,20 +13,24 @@
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput;
 import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity;
+import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput;
 import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
 import com.linkedin.metadata.utils.EntityKeyUtils;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.structured.PrimitivePropertyValue;
 import com.linkedin.structured.PropertyCardinality;
 import com.linkedin.structured.PropertyValue;
 import com.linkedin.structured.StructuredPropertyKey;
+import com.linkedin.structured.StructuredPropertySettings;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
-import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import javax.annotation.Nonnull;
 
@@ -54,40 +59,28 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
                   "Unable to create structured property. Please contact your admin.");
             }
             final StructuredPropertyKey key = new StructuredPropertyKey();
-            final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString();
+            final String id =
+                StructuredPropertyUtils.getPropertyId(input.getId(), input.getQualifiedName());
             key.setId(id);
             final Urn propertyUrn =
                 EntityKeyUtils.convertEntityKeyToUrn(key, STRUCTURED_PROPERTY_ENTITY_NAME);
-            StructuredPropertyDefinitionPatchBuilder builder =
-                new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn);
-
-            builder.setQualifiedName(input.getQualifiedName());
-            builder.setValueType(input.getValueType());
-            input.getEntityTypes().forEach(builder::addEntityType);
-            if (input.getDisplayName() != null) {
-              builder.setDisplayName(input.getDisplayName());
-            }
-            if (input.getDescription() != null) {
-              builder.setDescription(input.getDescription());
-            }
-            if (input.getImmutable() != null) {
-              builder.setImmutable(input.getImmutable());
-            }
-            if (input.getTypeQualifier() != null) {
-              buildTypeQualifier(input, builder);
-            }
-            if (input.getAllowedValues() != null) {
-              buildAllowedValues(input, builder);
+
+            if (_entityClient.exists(context.getOperationContext(), propertyUrn)) {
+              throw new IllegalArgumentException(
+                  "A structured property already exists with this urn");
             }
-            if (input.getCardinality() != null) {
-              builder.setCardinality(
-                  PropertyCardinality.valueOf(input.getCardinality().toString()));
+
+            List<MetadataChangeProposal> mcps = new ArrayList<>();
+
+            // first, create the property definition itself
+            mcps.add(createPropertyDefinition(context, propertyUrn, id, input));
+
+            // then add the settings aspect if we're adding any settings inputs
+            if (input.getSettings() != null) {
+              mcps.add(createPropertySettings(context, propertyUrn, input.getSettings()));
             }
-            builder.setCreated(context.getOperationContext().getAuditStamp());
-            builder.setLastModified(context.getOperationContext().getAuditStamp());
 
-            MetadataChangeProposal mcp = builder.build();
-            _entityClient.ingestProposal(context.getOperationContext(), mcp, false);
+            _entityClient.batchIngestProposals(context.getOperationContext(), mcps, false);
 
             EntityResponse response =
                 _entityClient.getV2(
@@ -103,6 +96,72 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
         });
   }
 
+  private MetadataChangeProposal createPropertySettings(
+      @Nonnull final QueryContext context,
+      @Nonnull final Urn propertyUrn,
+      final StructuredPropertySettingsInput settingsInput)
+      throws Exception {
+    StructuredPropertySettings settings = new StructuredPropertySettings();
+
+    if (settingsInput.getIsHidden() != null) {
+      settings.setIsHidden(settingsInput.getIsHidden());
+    }
+    if (settingsInput.getShowInSearchFilters() != null) {
+      settings.setShowInSearchFilters(settingsInput.getShowInSearchFilters());
+    }
+    if (settingsInput.getShowInAssetSummary() != null) {
+      settings.setShowInAssetSummary(settingsInput.getShowInAssetSummary());
+    }
+    if (settingsInput.getShowAsAssetBadge() != null) {
+      settings.setShowAsAssetBadge(settingsInput.getShowAsAssetBadge());
+    }
+    if (settingsInput.getShowInColumnsTable() != null) {
+      settings.setShowInColumnsTable(settingsInput.getShowInColumnsTable());
+    }
+    settings.setLastModified(context.getOperationContext().getAuditStamp());
+
+    StructuredPropertyUtils.validatePropertySettings(settings, true);
+
+    return buildMetadataChangeProposalWithUrn(
+        propertyUrn, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, settings);
+  }
+
+  private MetadataChangeProposal createPropertyDefinition(
+      @Nonnull final QueryContext context,
+      @Nonnull final Urn propertyUrn,
+      @Nonnull final String id,
+      final CreateStructuredPropertyInput input)
+      throws Exception {
+    StructuredPropertyDefinitionPatchBuilder builder =
+        new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn);
+
+    builder.setQualifiedName(id);
+    builder.setValueType(input.getValueType());
+    input.getEntityTypes().forEach(builder::addEntityType);
+    if (input.getDisplayName() != null) {
+      builder.setDisplayName(input.getDisplayName());
+    }
+    if (input.getDescription() != null) {
+      builder.setDescription(input.getDescription());
+    }
+    if (input.getImmutable() != null) {
+      builder.setImmutable(input.getImmutable());
+    }
+    if (input.getTypeQualifier() != null) {
+      buildTypeQualifier(input, builder);
+    }
+    if (input.getAllowedValues() != null) {
+      buildAllowedValues(input, builder);
+    }
+    if (input.getCardinality() != null) {
+      builder.setCardinality(PropertyCardinality.valueOf(input.getCardinality().toString()));
+    }
+    builder.setCreated(context.getOperationContext().getAuditStamp());
+    builder.setLastModified(context.getOperationContext().getAuditStamp());
+
+    return builder.build();
+  }
+
   private void buildTypeQualifier(
       @Nonnull final CreateStructuredPropertyInput input,
       @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) {
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java
index e7d59494654fd..58f8d340fcc07 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java
@@ -6,6 +6,7 @@
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
+import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput;
 import com.linkedin.entity.client.EntityClient;
@@ -42,6 +43,23 @@ public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment)
                   "Unable to delete structured property. Please contact your admin.");
             }
             _entityClient.deleteEntity(context.getOperationContext(), propertyUrn);
+            // Asynchronously Delete all references to the entity (to return quickly)
+            GraphQLConcurrencyUtils.supplyAsync(
+                () -> {
+                  try {
+                    _entityClient.deleteEntityReferences(
+                        context.getOperationContext(), propertyUrn);
+                  } catch (Exception e) {
+                    log.error(
+                        String.format(
+                            "Caught exception while attempting to clear all entity references for Structured Property with urn %s",
+                            propertyUrn),
+                        e);
+                  }
+                  return null;
+                },
+                this.getClass().getSimpleName(),
+                "get");
             return true;
           } catch (Exception e) {
             throw new RuntimeException(
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java
index ea8c6dac36a4a..313e0a16d8916 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java
@@ -93,7 +93,7 @@ public CompletableFuture<com.linkedin.datahub.graphql.generated.StructuredProper
                         .getValue()
                         .data());
 
-            return StructuredPropertiesMapper.map(context, structuredProperties);
+            return StructuredPropertiesMapper.map(context, structuredProperties, assetUrn);
           } catch (Exception e) {
             throw new RuntimeException(
                 String.format("Failed to perform update against input %s", input), e);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java
index f5c101ba2bf64..6bbe2eec5e1e2 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java
@@ -1,8 +1,8 @@
 package com.linkedin.datahub.graphql.resolvers.structuredproperties;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
-import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
-import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
+import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithUrn;
+import static com.linkedin.metadata.Constants.*;
 
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
@@ -13,18 +13,23 @@
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity;
+import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput;
 import com.linkedin.datahub.graphql.generated.UpdateStructuredPropertyInput;
 import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.structured.PrimitivePropertyValue;
 import com.linkedin.structured.PropertyCardinality;
 import com.linkedin.structured.PropertyValue;
 import com.linkedin.structured.StructuredPropertyDefinition;
+import com.linkedin.structured.StructuredPropertySettings;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import javax.annotation.Nonnull;
@@ -57,36 +62,24 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
                   "Unable to update structured property. Please contact your admin.");
             }
             final Urn propertyUrn = UrnUtils.getUrn(input.getUrn());
-            StructuredPropertyDefinition existingDefinition =
+            final EntityResponse entityResponse =
                 getExistingStructuredProperty(context, propertyUrn);
-            StructuredPropertyDefinitionPatchBuilder builder =
-                new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn);
 
-            if (input.getDisplayName() != null) {
-              builder.setDisplayName(input.getDisplayName());
-            }
-            if (input.getDescription() != null) {
-              builder.setDescription(input.getDescription());
-            }
-            if (input.getImmutable() != null) {
-              builder.setImmutable(input.getImmutable());
-            }
-            if (input.getTypeQualifier() != null) {
-              buildTypeQualifier(input, builder, existingDefinition);
-            }
-            if (input.getNewAllowedValues() != null) {
-              buildAllowedValues(input, builder);
-            }
-            if (input.getSetCardinalityAsMultiple() != null) {
-              builder.setCardinality(PropertyCardinality.MULTIPLE);
+            List<MetadataChangeProposal> mcps = new ArrayList<>();
+
+            // first update the definition aspect if we need to
+            MetadataChangeProposal definitionMcp =
+                updateDefinition(input, context, propertyUrn, entityResponse);
+            if (definitionMcp != null) {
+              mcps.add(definitionMcp);
             }
-            if (input.getNewEntityTypes() != null) {
-              input.getNewEntityTypes().forEach(builder::addEntityType);
+
+            // then update the settings aspect if we need to
+            if (input.getSettings() != null) {
+              mcps.add(updateSettings(context, input.getSettings(), propertyUrn, entityResponse));
             }
-            builder.setLastModified(context.getOperationContext().getAuditStamp());
 
-            MetadataChangeProposal mcp = builder.build();
-            _entityClient.ingestProposal(context.getOperationContext(), mcp, false);
+            _entityClient.batchIngestProposals(context.getOperationContext(), mcps, false);
 
             EntityResponse response =
                 _entityClient.getV2(
@@ -102,6 +95,120 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
         });
   }
 
+  private boolean hasSettingsChanged(
+      StructuredPropertySettings existingSettings, StructuredPropertySettingsInput settingsInput) {
+    if (settingsInput.getIsHidden() != null
+        && !existingSettings.isIsHidden().equals(settingsInput.getIsHidden())) {
+      return true;
+    }
+    if (settingsInput.getShowInSearchFilters() != null
+        && !existingSettings
+            .isShowInSearchFilters()
+            .equals(settingsInput.getShowInSearchFilters())) {
+      return true;
+    }
+    if (settingsInput.getShowInAssetSummary() != null
+        && !existingSettings.isShowInAssetSummary().equals(settingsInput.getShowInAssetSummary())) {
+      return true;
+    }
+    if (settingsInput.getShowAsAssetBadge() != null
+        && !existingSettings.isShowAsAssetBadge().equals(settingsInput.getShowAsAssetBadge())) {
+      return true;
+    }
+    if (settingsInput.getShowInColumnsTable() != null
+        && !existingSettings.isShowInColumnsTable().equals(settingsInput.getShowInColumnsTable())) {
+      return true;
+    }
+    return false;
+  }
+
+  private MetadataChangeProposal updateSettings(
+      @Nonnull final QueryContext context,
+      @Nonnull final StructuredPropertySettingsInput settingsInput,
+      @Nonnull final Urn propertyUrn,
+      @Nonnull final EntityResponse entityResponse)
+      throws Exception {
+    StructuredPropertySettings existingSettings =
+        getExistingStructuredPropertySettings(entityResponse);
+    // check if settings has changed to determine if we should update the timestamp
+    boolean hasChanged = hasSettingsChanged(existingSettings, settingsInput);
+    if (hasChanged) {
+      existingSettings.setLastModified(context.getOperationContext().getAuditStamp());
+    }
+
+    if (settingsInput.getIsHidden() != null) {
+      existingSettings.setIsHidden(settingsInput.getIsHidden());
+    }
+    if (settingsInput.getShowInSearchFilters() != null) {
+      existingSettings.setShowInSearchFilters(settingsInput.getShowInSearchFilters());
+    }
+    if (settingsInput.getShowInAssetSummary() != null) {
+      existingSettings.setShowInAssetSummary(settingsInput.getShowInAssetSummary());
+    }
+    if (settingsInput.getShowAsAssetBadge() != null) {
+      existingSettings.setShowAsAssetBadge(settingsInput.getShowAsAssetBadge());
+    }
+    if (settingsInput.getShowInColumnsTable() != null) {
+      existingSettings.setShowInColumnsTable(settingsInput.getShowInColumnsTable());
+    }
+
+    StructuredPropertyUtils.validatePropertySettings(existingSettings, true);
+
+    return buildMetadataChangeProposalWithUrn(
+        propertyUrn, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, existingSettings);
+  }
+
+  private MetadataChangeProposal updateDefinition(
+      @Nonnull final UpdateStructuredPropertyInput input,
+      @Nonnull final QueryContext context,
+      @Nonnull final Urn propertyUrn,
+      @Nonnull final EntityResponse entityResponse)
+      throws Exception {
+    StructuredPropertyDefinition existingDefinition =
+        getExistingStructuredPropertyDefinition(entityResponse);
+    StructuredPropertyDefinitionPatchBuilder builder =
+        new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn);
+
+    boolean hasUpdatedDefinition = false;
+
+    if (input.getDisplayName() != null) {
+      builder.setDisplayName(input.getDisplayName());
+      hasUpdatedDefinition = true;
+    }
+    if (input.getDescription() != null) {
+      builder.setDescription(input.getDescription());
+      hasUpdatedDefinition = true;
+    }
+    if (input.getImmutable() != null) {
+      builder.setImmutable(input.getImmutable());
+      hasUpdatedDefinition = true;
+    }
+    if (input.getTypeQualifier() != null) {
+      buildTypeQualifier(input, builder, existingDefinition);
+      hasUpdatedDefinition = true;
+    }
+    if (input.getNewAllowedValues() != null) {
+      buildAllowedValues(input, builder);
+      hasUpdatedDefinition = true;
+    }
+    if (input.getSetCardinalityAsMultiple() != null
+        && input.getSetCardinalityAsMultiple().equals(true)) {
+      builder.setCardinality(PropertyCardinality.MULTIPLE);
+      hasUpdatedDefinition = true;
+    }
+    if (input.getNewEntityTypes() != null) {
+      input.getNewEntityTypes().forEach(builder::addEntityType);
+      hasUpdatedDefinition = true;
+    }
+
+    if (hasUpdatedDefinition) {
+      builder.setLastModified(context.getOperationContext().getAuditStamp());
+
+      return builder.build();
+    }
+    return null;
+  }
+
   private void buildTypeQualifier(
       @Nonnull final UpdateStructuredPropertyInput input,
       @Nonnull final StructuredPropertyDefinitionPatchBuilder builder,
@@ -141,17 +248,40 @@ private void buildAllowedValues(
             });
   }
 
-  private StructuredPropertyDefinition getExistingStructuredProperty(
+  private EntityResponse getExistingStructuredProperty(
       @Nonnull final QueryContext context, @Nonnull final Urn propertyUrn) throws Exception {
-    EntityResponse response =
-        _entityClient.getV2(
-            context.getOperationContext(), STRUCTURED_PROPERTY_ENTITY_NAME, propertyUrn, null);
+    return _entityClient.getV2(
+        context.getOperationContext(), STRUCTURED_PROPERTY_ENTITY_NAME, propertyUrn, null);
+  }
 
+  private StructuredPropertyDefinition getExistingStructuredPropertyDefinition(
+      EntityResponse response) throws Exception {
     if (response != null
         && response.getAspects().containsKey(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) {
       return new StructuredPropertyDefinition(
-          response.getAspects().get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME).getValue().data());
+          response
+              .getAspects()
+              .get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)
+              .getValue()
+              .data()
+              .copy());
     }
-    return null;
+    throw new IllegalArgumentException(
+        "Attempting to update a structured property with no definition aspect.");
+  }
+
+  private StructuredPropertySettings getExistingStructuredPropertySettings(EntityResponse response)
+      throws Exception {
+    if (response != null
+        && response.getAspects().containsKey(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME)) {
+      return new StructuredPropertySettings(
+          response
+              .getAspects()
+              .get(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME)
+              .getValue()
+              .data()
+              .copy());
+    }
+    return new StructuredPropertySettings();
   }
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java
index 770c8a0d749c3..6c1d7949332fb 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java
@@ -103,7 +103,7 @@ public CompletableFuture<com.linkedin.datahub.graphql.generated.StructuredProper
             _entityClient.ingestProposal(
                 context.getOperationContext(), structuredPropertiesProposal, false);
 
-            return StructuredPropertiesMapper.map(context, structuredProperties);
+            return StructuredPropertiesMapper.map(context, structuredProperties, assetUrn);
           } catch (Exception e) {
             throw new RuntimeException(
                 String.format("Failed to perform update against input %s", input), e);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java
new file mode 100644
index 0000000000000..7ae719a23b00a
--- /dev/null
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java
@@ -0,0 +1,25 @@
+package com.linkedin.datahub.graphql.resolvers.type;
+
+import com.linkedin.datahub.graphql.generated.CorpGroup;
+import com.linkedin.datahub.graphql.generated.CorpUser;
+import graphql.TypeResolutionEnvironment;
+import graphql.schema.GraphQLObjectType;
+import graphql.schema.TypeResolver;
+
+public class ResolvedActorResolver implements TypeResolver {
+
+  public static final String CORP_USER = "CorpUser";
+  public static final String CORP_GROUP = "CorpGroup";
+
+  @Override
+  public GraphQLObjectType getType(TypeResolutionEnvironment env) {
+    if (env.getObject() instanceof CorpUser) {
+      return env.getSchema().getObjectType(CORP_USER);
+    } else if (env.getObject() instanceof CorpGroup) {
+      return env.getSchema().getObjectType(CORP_GROUP);
+    } else {
+      throw new RuntimeException(
+          "Unrecognized object type provided to type resolver, Type:" + env.getObject().toString());
+    }
+  }
+}
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java
index 561c3b9bec1e0..e23eaa9e5c1ec 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java
@@ -142,7 +142,8 @@ public Chart apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((chart, dataMap) ->
             chart.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java
index 7c6de02ecc876..9781643c414c8 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java
@@ -28,6 +28,7 @@ public InstitutionalMemoryMetadata apply(
     result.setDescription(input.getDescription()); // deprecated field
     result.setLabel(input.getDescription());
     result.setAuthor(getAuthor(input.getCreateStamp().getActor().toString()));
+    result.setActor(ResolvedActorMapper.map(input.getCreateStamp().getActor()));
     result.setCreated(AuditStampMapper.map(context, input.getCreateStamp()));
     result.setAssociatedUrn(entityUrn.toString());
     return result;
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java
new file mode 100644
index 0000000000000..c00ffd0b828b1
--- /dev/null
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java
@@ -0,0 +1,31 @@
+package com.linkedin.datahub.graphql.types.common.mappers;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.graphql.generated.CorpGroup;
+import com.linkedin.datahub.graphql.generated.CorpUser;
+import com.linkedin.datahub.graphql.generated.EntityType;
+import com.linkedin.datahub.graphql.generated.ResolvedActor;
+import com.linkedin.metadata.Constants;
+import javax.annotation.Nonnull;
+
+public class ResolvedActorMapper {
+
+  public static final ResolvedActorMapper INSTANCE = new ResolvedActorMapper();
+
+  public static ResolvedActor map(@Nonnull final Urn actorUrn) {
+    return INSTANCE.apply(actorUrn);
+  }
+
+  public ResolvedActor apply(@Nonnull final Urn actorUrn) {
+    if (actorUrn.getEntityType().equals(Constants.CORP_GROUP_ENTITY_NAME)) {
+      CorpGroup partialGroup = new CorpGroup();
+      partialGroup.setUrn(actorUrn.toString());
+      partialGroup.setType(EntityType.CORP_GROUP);
+      return partialGroup;
+    }
+    CorpUser partialUser = new CorpUser();
+    partialUser.setUrn(actorUrn.toString());
+    partialUser.setType(EntityType.CORP_USER);
+    return (ResolvedActor) partialUser;
+  }
+}
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java
index db44a5be8bdd3..37b021fcb1091 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java
@@ -52,7 +52,8 @@ public class ContainerType
           Constants.DEPRECATION_ASPECT_NAME,
           Constants.DATA_PRODUCTS_ASPECT_NAME,
           Constants.STRUCTURED_PROPERTIES_ASPECT_NAME,
-          Constants.FORMS_ASPECT_NAME);
+          Constants.FORMS_ASPECT_NAME,
+          Constants.ACCESS_ASPECT_NAME);
 
   private static final Set<String> FACET_FIELDS = ImmutableSet.of("origin", "platform");
   private static final String ENTITY_NAME = "container";
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java
index 2c0dc142bee3d..7ac00c46475bc 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java
@@ -2,6 +2,7 @@
 
 import static com.linkedin.metadata.Constants.*;
 
+import com.linkedin.common.Access;
 import com.linkedin.common.DataPlatformInstance;
 import com.linkedin.common.Deprecation;
 import com.linkedin.common.Forms;
@@ -30,6 +31,7 @@
 import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
 import com.linkedin.datahub.graphql.types.form.FormsMapper;
 import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
+import com.linkedin.datahub.graphql.types.rolemetadata.mappers.AccessMapper;
 import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper;
 import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper;
 import com.linkedin.domain.Domains;
@@ -105,6 +107,11 @@ public static Container map(
               context, new GlossaryTerms(envelopedTerms.getValue().data()), entityUrn));
     }
 
+    final EnvelopedAspect accessAspect = aspects.get(ACCESS_ASPECT_NAME);
+    if (accessAspect != null) {
+      result.setAccess(AccessMapper.map(new Access(accessAspect.getValue().data()), entityUrn));
+    }
+
     final EnvelopedAspect envelopedInstitutionalMemory =
         aspects.get(Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME);
     if (envelopedInstitutionalMemory != null) {
@@ -154,7 +161,9 @@ public static Container map(
     if (envelopedStructuredProps != null) {
       result.setStructuredProperties(
           StructuredPropertiesMapper.map(
-              context, new StructuredProperties(envelopedStructuredProps.getValue().data())));
+              context,
+              new StructuredProperties(envelopedStructuredProps.getValue().data()),
+              entityUrn));
     }
 
     final EnvelopedAspect envelopedForms = aspects.get(FORMS_ASPECT_NAME);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java
index 6246cf64bbf7f..010816431f54d 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java
@@ -59,7 +59,8 @@ public CorpGroup apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((entity, dataMap) ->
             entity.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java
index 4fa278983399b..a94b555daebdf 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java
@@ -88,7 +88,8 @@ public CorpUser apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((entity, dataMap) ->
             entity.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java
index 4fa52b1136564..fd1c7a5db2a79 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java
@@ -142,7 +142,8 @@ public Dashboard apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((dashboard, dataMap) ->
             dashboard.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java
index 9e2612f60abda..44bc6a99eae4b 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java
@@ -114,7 +114,8 @@ public DataFlow apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((entity, dataMap) ->
             entity.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java
index d7da875bc2a29..772871d77f217 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java
@@ -135,7 +135,8 @@ public DataJob apply(
                 result.setSubTypes(SubTypesMapper.map(context, new SubTypes(data)));
               } else if (STRUCTURED_PROPERTIES_ASPECT_NAME.equals(name)) {
                 result.setStructuredProperties(
-                    StructuredPropertiesMapper.map(context, new StructuredProperties(data)));
+                    StructuredPropertiesMapper.map(
+                        context, new StructuredProperties(data), entityUrn));
               } else if (FORMS_ASPECT_NAME.equals(name)) {
                 result.setForms(FormsMapper.map(new Forms(data), entityUrn.toString()));
               }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java
index 08637dbfd01ed..8693ec97f1a2e 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java
@@ -92,7 +92,8 @@ public DataProduct apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((entity, dataMap) ->
             entity.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java
index 65b5d39e31569..6a3f9cb9b21f3 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java
@@ -86,7 +86,7 @@ public class DatasetType
           EMBED_ASPECT_NAME,
           DATA_PRODUCTS_ASPECT_NAME,
           BROWSE_PATHS_V2_ASPECT_NAME,
-          ACCESS_DATASET_ASPECT_NAME,
+          ACCESS_ASPECT_NAME,
           STRUCTURED_PROPERTIES_ASPECT_NAME,
           FORMS_ASPECT_NAME,
           SUB_TYPES_ASPECT_NAME);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java
index a7b5f6de0c183..e411014c23c89 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java
@@ -166,14 +166,15 @@ public Dataset apply(
         (dataset, dataMap) ->
             dataset.setBrowsePathV2(BrowsePathsV2Mapper.map(context, new BrowsePathsV2(dataMap))));
     mappingHelper.mapToResult(
-        ACCESS_DATASET_ASPECT_NAME,
+        ACCESS_ASPECT_NAME,
         ((dataset, dataMap) ->
             dataset.setAccess(AccessMapper.map(new Access(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((entity, dataMap) ->
             entity.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((dataset, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java
index 7d05e0862a96d..ffcb94a0b7e29 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java
@@ -71,7 +71,9 @@ public static Domain map(@Nullable QueryContext context, final EntityResponse en
     if (envelopedStructuredProps != null) {
       result.setStructuredProperties(
           StructuredPropertiesMapper.map(
-              context, new StructuredProperties(envelopedStructuredProps.getValue().data())));
+              context,
+              new StructuredProperties(envelopedStructuredProps.getValue().data()),
+              entityUrn));
     }
 
     final EnvelopedAspect envelopedForms = aspects.get(FORMS_ASPECT_NAME);
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java
index 4912d18614f41..a694b62999080 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java
@@ -59,7 +59,8 @@ public GlossaryNode apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((entity, dataMap) ->
             entity.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java
index 1274646f45ec4..e309ffad84df5 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java
@@ -90,7 +90,8 @@ public GlossaryTerm apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((entity, dataMap) ->
             entity.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/EmbeddedModelMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/EmbeddedModelMapper.java
new file mode 100644
index 0000000000000..62e7c90ab9b0e
--- /dev/null
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/EmbeddedModelMapper.java
@@ -0,0 +1,12 @@
+package com.linkedin.datahub.graphql.types.mappers;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.graphql.QueryContext;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/** Made for models that are embedded in other models and thus do not encode their own URN. */
+public interface EmbeddedModelMapper<I, O> {
+  O apply(
+      @Nullable final QueryContext context, @Nonnull final I input, @Nonnull final Urn entityUrn);
+}
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java
index 0d69e62c621a6..8fe58df2d2ede 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java
@@ -70,6 +70,9 @@ public static FacetMetadata mapFacet(
         aggregationFacets.stream()
             .map(facet -> facet.equals("entity") || facet.contains("_entityType"))
             .collect(Collectors.toList());
+    if (aggregationMetadata.getEntity() != null) {
+      facetMetadata.setEntity(UrnToEntityMapper.map(context, aggregationMetadata.getEntity()));
+    }
     facetMetadata.setField(aggregationMetadata.getName());
     facetMetadata.setDisplayName(
         Optional.ofNullable(aggregationMetadata.getDisplayName())
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java
index a4f3aa7a0e226..74076fd2f4ee9 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java
@@ -75,7 +75,8 @@ public MLFeature apply(
             mlFeature.setOwnership(
                 OwnershipMapper.map(context, new Ownership(dataMap), entityUrn)));
     mappingHelper.mapToResult(
-        context, ML_FEATURE_PROPERTIES_ASPECT_NAME, MLFeatureMapper::mapMLFeatureProperties);
+        ML_FEATURE_PROPERTIES_ASPECT_NAME,
+        (entity, dataMap) -> mapMLFeatureProperties(context, entity, dataMap, entityUrn));
     mappingHelper.mapToResult(
         INSTITUTIONAL_MEMORY_ASPECT_NAME,
         (mlFeature, dataMap) ->
@@ -115,7 +116,8 @@ public MLFeature apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((mlFeature, dataMap) ->
             mlFeature.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
@@ -137,10 +139,13 @@ private static void mapMLFeatureKey(@Nonnull MLFeature mlFeature, @Nonnull DataM
   private static void mapMLFeatureProperties(
       @Nullable final QueryContext context,
       @Nonnull MLFeature mlFeature,
-      @Nonnull DataMap dataMap) {
+      @Nonnull DataMap dataMap,
+      @Nonnull Urn entityUrn) {
     MLFeatureProperties featureProperties = new MLFeatureProperties(dataMap);
-    mlFeature.setFeatureProperties(MLFeaturePropertiesMapper.map(context, featureProperties));
-    mlFeature.setProperties(MLFeaturePropertiesMapper.map(context, featureProperties));
+    com.linkedin.datahub.graphql.generated.MLFeatureProperties graphqlProperties =
+        MLFeaturePropertiesMapper.map(context, featureProperties, entityUrn);
+    mlFeature.setFeatureProperties(graphqlProperties);
+    mlFeature.setProperties(graphqlProperties);
     mlFeature.setDescription(featureProperties.getDescription());
     if (featureProperties.getDataType() != null) {
       mlFeature.setDataType(MLFeatureDataType.valueOf(featureProperties.getDataType().toString()));
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeaturePropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeaturePropertiesMapper.java
index 92d090275867d..08ac3a1b5f138 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeaturePropertiesMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeaturePropertiesMapper.java
@@ -1,29 +1,34 @@
 package com.linkedin.datahub.graphql.types.mlmodel.mappers;
 
+import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.Dataset;
 import com.linkedin.datahub.graphql.generated.MLFeatureDataType;
 import com.linkedin.datahub.graphql.generated.MLFeatureProperties;
-import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
+import com.linkedin.datahub.graphql.types.mappers.EmbeddedModelMapper;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.NonNull;
 
 public class MLFeaturePropertiesMapper
-    implements ModelMapper<com.linkedin.ml.metadata.MLFeatureProperties, MLFeatureProperties> {
+    implements EmbeddedModelMapper<
+        com.linkedin.ml.metadata.MLFeatureProperties, MLFeatureProperties> {
 
   public static final MLFeaturePropertiesMapper INSTANCE = new MLFeaturePropertiesMapper();
 
   public static MLFeatureProperties map(
       @Nullable QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLFeatureProperties mlFeatureProperties) {
-    return INSTANCE.apply(context, mlFeatureProperties);
+      @Nonnull final com.linkedin.ml.metadata.MLFeatureProperties mlFeatureProperties,
+      @Nonnull Urn entityUrn) {
+    return INSTANCE.apply(context, mlFeatureProperties, entityUrn);
   }
 
   @Override
   public MLFeatureProperties apply(
       @Nullable QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLFeatureProperties mlFeatureProperties) {
+      @Nonnull final com.linkedin.ml.metadata.MLFeatureProperties mlFeatureProperties,
+      @Nonnull Urn entityUrn) {
     final MLFeatureProperties result = new MLFeatureProperties();
 
     result.setDescription(mlFeatureProperties.getDescription());
@@ -45,6 +50,9 @@ public MLFeatureProperties apply(
               .collect(Collectors.toList()));
     }
 
+    result.setCustomProperties(
+        CustomPropertiesMapper.map(mlFeatureProperties.getCustomProperties(), entityUrn));
+
     return result;
   }
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java
index 30bf4dda1cf4f..65bc8e84f7bbb 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java
@@ -76,7 +76,7 @@ public MLFeatureTable apply(
     mappingHelper.mapToResult(ML_FEATURE_TABLE_KEY_ASPECT_NAME, this::mapMLFeatureTableKey);
     mappingHelper.mapToResult(
         ML_FEATURE_TABLE_PROPERTIES_ASPECT_NAME,
-        (entity, dataMap) -> this.mapMLFeatureTableProperties(context, entity, dataMap, entityUrn));
+        (entity, dataMap) -> mapMLFeatureTableProperties(context, entity, dataMap, entityUrn));
     mappingHelper.mapToResult(
         INSTITUTIONAL_MEMORY_ASPECT_NAME,
         (mlFeatureTable, dataMap) ->
@@ -117,7 +117,8 @@ public MLFeatureTable apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((mlFeatureTable, dataMap) ->
             mlFeatureTable.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
@@ -145,10 +146,10 @@ private static void mapMLFeatureTableProperties(
       @Nonnull DataMap dataMap,
       Urn entityUrn) {
     MLFeatureTableProperties featureTableProperties = new MLFeatureTableProperties(dataMap);
-    mlFeatureTable.setFeatureTableProperties(
-        MLFeatureTablePropertiesMapper.map(context, featureTableProperties, entityUrn));
-    mlFeatureTable.setProperties(
-        MLFeatureTablePropertiesMapper.map(context, featureTableProperties, entityUrn));
+    com.linkedin.datahub.graphql.generated.MLFeatureTableProperties graphqlProperties =
+        MLFeatureTablePropertiesMapper.map(context, featureTableProperties, entityUrn);
+    mlFeatureTable.setFeatureTableProperties(graphqlProperties);
+    mlFeatureTable.setProperties(graphqlProperties);
     mlFeatureTable.setDescription(featureTableProperties.getDescription());
   }
 
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTablePropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTablePropertiesMapper.java
index d9fed13ed0d0b..3c054cb6a9a5b 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTablePropertiesMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTablePropertiesMapper.java
@@ -8,26 +8,30 @@
 import com.linkedin.datahub.graphql.generated.MLFeatureTableProperties;
 import com.linkedin.datahub.graphql.generated.MLPrimaryKey;
 import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
+import com.linkedin.datahub.graphql.types.mappers.EmbeddedModelMapper;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.NonNull;
 
-public class MLFeatureTablePropertiesMapper {
+public class MLFeatureTablePropertiesMapper
+    implements EmbeddedModelMapper<
+        com.linkedin.ml.metadata.MLFeatureTableProperties, MLFeatureTableProperties> {
 
   public static final MLFeatureTablePropertiesMapper INSTANCE =
       new MLFeatureTablePropertiesMapper();
 
   public static MLFeatureTableProperties map(
       @Nullable final QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLFeatureTableProperties mlFeatureTableProperties,
-      Urn entityUrn) {
+      @Nonnull final com.linkedin.ml.metadata.MLFeatureTableProperties mlFeatureTableProperties,
+      @Nonnull Urn entityUrn) {
     return INSTANCE.apply(context, mlFeatureTableProperties, entityUrn);
   }
 
-  public static MLFeatureTableProperties apply(
+  @Override
+  public MLFeatureTableProperties apply(
       @Nullable final QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLFeatureTableProperties mlFeatureTableProperties,
-      Urn entityUrn) {
+      @Nonnull final com.linkedin.ml.metadata.MLFeatureTableProperties mlFeatureTableProperties,
+      @Nonnull Urn entityUrn) {
     final MLFeatureTableProperties result = new MLFeatureTableProperties();
 
     result.setDescription(mlFeatureTableProperties.getDescription());
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java
index 7e99040e44c82..9009972a47616 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java
@@ -75,9 +75,8 @@ public MLModelGroup apply(
     mappingHelper.mapToResult(
         ML_MODEL_GROUP_KEY_ASPECT_NAME, MLModelGroupMapper::mapToMLModelGroupKey);
     mappingHelper.mapToResult(
-        context,
         ML_MODEL_GROUP_PROPERTIES_ASPECT_NAME,
-        MLModelGroupMapper::mapToMLModelGroupProperties);
+        (entity, dataMap) -> mapToMLModelGroupProperties(context, entity, dataMap, entityUrn));
     mappingHelper.mapToResult(
         STATUS_ASPECT_NAME,
         (mlModelGroup, dataMap) ->
@@ -112,7 +111,8 @@ public MLModelGroup apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((mlModelGroup, dataMap) ->
             mlModelGroup.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
@@ -135,9 +135,13 @@ private static void mapToMLModelGroupKey(MLModelGroup mlModelGroup, DataMap data
   }
 
   private static void mapToMLModelGroupProperties(
-      @Nullable final QueryContext context, MLModelGroup mlModelGroup, DataMap dataMap) {
+      @Nullable final QueryContext context,
+      MLModelGroup mlModelGroup,
+      DataMap dataMap,
+      @Nonnull Urn entityUrn) {
     MLModelGroupProperties modelGroupProperties = new MLModelGroupProperties(dataMap);
-    mlModelGroup.setProperties(MLModelGroupPropertiesMapper.map(context, modelGroupProperties));
+    mlModelGroup.setProperties(
+        MLModelGroupPropertiesMapper.map(context, modelGroupProperties, entityUrn));
     if (modelGroupProperties.getDescription() != null) {
       mlModelGroup.setDescription(modelGroupProperties.getDescription());
     }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java
index 9f1918f9ec489..a6cfded9865d9 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java
@@ -1,27 +1,31 @@
 package com.linkedin.datahub.graphql.types.mlmodel.mappers;
 
+import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.MLModelGroupProperties;
-import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
+import com.linkedin.datahub.graphql.types.mappers.EmbeddedModelMapper;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.NonNull;
 
 public class MLModelGroupPropertiesMapper
-    implements ModelMapper<
+    implements EmbeddedModelMapper<
         com.linkedin.ml.metadata.MLModelGroupProperties, MLModelGroupProperties> {
 
   public static final MLModelGroupPropertiesMapper INSTANCE = new MLModelGroupPropertiesMapper();
 
   public static MLModelGroupProperties map(
       @Nullable QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLModelGroupProperties mlModelGroupProperties) {
-    return INSTANCE.apply(context, mlModelGroupProperties);
+      @Nonnull final com.linkedin.ml.metadata.MLModelGroupProperties mlModelGroupProperties,
+      @Nonnull Urn entityUrn) {
+    return INSTANCE.apply(context, mlModelGroupProperties, entityUrn);
   }
 
   @Override
   public MLModelGroupProperties apply(
       @Nullable QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLModelGroupProperties mlModelGroupProperties) {
+      @Nonnull final com.linkedin.ml.metadata.MLModelGroupProperties mlModelGroupProperties,
+      @Nonnull Urn entityUrn) {
     final MLModelGroupProperties result = new MLModelGroupProperties();
 
     result.setDescription(mlModelGroupProperties.getDescription());
@@ -30,6 +34,9 @@ public MLModelGroupProperties apply(
     }
     result.setCreatedAt(mlModelGroupProperties.getCreatedAt());
 
+    result.setCustomProperties(
+        CustomPropertiesMapper.map(mlModelGroupProperties.getCustomProperties(), entityUrn));
+
     return result;
   }
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java
index a3bc5c663c89a..7102fd4aed974 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java
@@ -174,7 +174,8 @@ public MLModel apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((dataset, dataMap) ->
             dataset.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java
index a89904b3ab915..265005c2caa9e 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java
@@ -7,25 +7,27 @@
 import com.linkedin.datahub.graphql.generated.MLModelGroup;
 import com.linkedin.datahub.graphql.generated.MLModelProperties;
 import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
+import com.linkedin.datahub.graphql.types.mappers.EmbeddedModelMapper;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.NonNull;
 
-public class MLModelPropertiesMapper {
+public class MLModelPropertiesMapper
+    implements EmbeddedModelMapper<com.linkedin.ml.metadata.MLModelProperties, MLModelProperties> {
 
   public static final MLModelPropertiesMapper INSTANCE = new MLModelPropertiesMapper();
 
   public static MLModelProperties map(
       @Nullable final QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLModelProperties mlModelProperties,
+      @Nonnull final com.linkedin.ml.metadata.MLModelProperties mlModelProperties,
       Urn entityUrn) {
     return INSTANCE.apply(context, mlModelProperties, entityUrn);
   }
 
   public MLModelProperties apply(
       @Nullable final QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLModelProperties mlModelProperties,
-      Urn entityUrn) {
+      @Nonnull final com.linkedin.ml.metadata.MLModelProperties mlModelProperties,
+      @Nonnull Urn entityUrn) {
     final MLModelProperties result = new MLModelProperties();
 
     result.setDate(mlModelProperties.getDate());
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java
index 36784f96ea30e..d48d93ede9c1a 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java
@@ -74,9 +74,8 @@ public MLPrimaryKey apply(
     mappingHelper.mapToResult(
         ML_PRIMARY_KEY_KEY_ASPECT_NAME, MLPrimaryKeyMapper::mapMLPrimaryKeyKey);
     mappingHelper.mapToResult(
-        context,
         ML_PRIMARY_KEY_PROPERTIES_ASPECT_NAME,
-        MLPrimaryKeyMapper::mapMLPrimaryKeyProperties);
+        (entity, dataMap) -> mapMLPrimaryKeyProperties(context, entity, dataMap, entityUrn));
     mappingHelper.mapToResult(
         INSTITUTIONAL_MEMORY_ASPECT_NAME,
         (mlPrimaryKey, dataMap) ->
@@ -112,7 +111,8 @@ public MLPrimaryKey apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((entity, dataMap) ->
             entity.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         FORMS_ASPECT_NAME,
         ((entity, dataMap) ->
@@ -131,11 +131,15 @@ private static void mapMLPrimaryKeyKey(MLPrimaryKey mlPrimaryKey, DataMap dataMa
   }
 
   private static void mapMLPrimaryKeyProperties(
-      @Nullable final QueryContext context, MLPrimaryKey mlPrimaryKey, DataMap dataMap) {
+      @Nullable final QueryContext context,
+      MLPrimaryKey mlPrimaryKey,
+      DataMap dataMap,
+      @Nonnull Urn entityUrn) {
     MLPrimaryKeyProperties primaryKeyProperties = new MLPrimaryKeyProperties(dataMap);
-    mlPrimaryKey.setPrimaryKeyProperties(
-        MLPrimaryKeyPropertiesMapper.map(context, primaryKeyProperties));
-    mlPrimaryKey.setProperties(MLPrimaryKeyPropertiesMapper.map(context, primaryKeyProperties));
+    com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties graphqlProperties =
+        MLPrimaryKeyPropertiesMapper.map(context, primaryKeyProperties, entityUrn);
+    mlPrimaryKey.setPrimaryKeyProperties(graphqlProperties);
+    mlPrimaryKey.setProperties(graphqlProperties);
     mlPrimaryKey.setDescription(primaryKeyProperties.getDescription());
     if (primaryKeyProperties.getDataType() != null) {
       mlPrimaryKey.setDataType(
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyPropertiesMapper.java
index 09e41fe7ee4e8..0bbe8f53f3271 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyPropertiesMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyPropertiesMapper.java
@@ -1,30 +1,34 @@
 package com.linkedin.datahub.graphql.types.mlmodel.mappers;
 
+import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.Dataset;
 import com.linkedin.datahub.graphql.generated.MLFeatureDataType;
 import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties;
-import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
+import com.linkedin.datahub.graphql.types.mappers.EmbeddedModelMapper;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.NonNull;
 
 public class MLPrimaryKeyPropertiesMapper
-    implements ModelMapper<
+    implements EmbeddedModelMapper<
         com.linkedin.ml.metadata.MLPrimaryKeyProperties, MLPrimaryKeyProperties> {
 
   public static final MLPrimaryKeyPropertiesMapper INSTANCE = new MLPrimaryKeyPropertiesMapper();
 
   public static MLPrimaryKeyProperties map(
       @Nullable QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLPrimaryKeyProperties mlPrimaryKeyProperties) {
-    return INSTANCE.apply(context, mlPrimaryKeyProperties);
+      @Nonnull final com.linkedin.ml.metadata.MLPrimaryKeyProperties mlPrimaryKeyProperties,
+      @Nonnull Urn entityUrn) {
+    return INSTANCE.apply(context, mlPrimaryKeyProperties, entityUrn);
   }
 
   @Override
   public MLPrimaryKeyProperties apply(
       @Nullable QueryContext context,
-      @NonNull final com.linkedin.ml.metadata.MLPrimaryKeyProperties mlPrimaryKeyProperties) {
+      @Nonnull final com.linkedin.ml.metadata.MLPrimaryKeyProperties mlPrimaryKeyProperties,
+      @Nonnull Urn entityUrn) {
     final MLPrimaryKeyProperties result = new MLPrimaryKeyProperties();
 
     result.setDescription(mlPrimaryKeyProperties.getDescription());
@@ -45,6 +49,9 @@ public MLPrimaryKeyProperties apply(
                 })
             .collect(Collectors.toList()));
 
+    result.setCustomProperties(
+        CustomPropertiesMapper.map(mlPrimaryKeyProperties.getCustomProperties(), entityUrn));
+
     return result;
   }
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java
index b1f27357d4550..30eac54aede9b 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java
@@ -41,7 +41,8 @@ public SchemaFieldEntity apply(
         STRUCTURED_PROPERTIES_ASPECT_NAME,
         ((schemaField, dataMap) ->
             schemaField.setStructuredProperties(
-                StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap)))));
+                StructuredPropertiesMapper.map(
+                    context, new StructuredProperties(dataMap), entityUrn))));
     mappingHelper.mapToResult(
         BUSINESS_ATTRIBUTE_ASPECT,
         (((schemaField, dataMap) ->
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java
index dc1ff7ca32971..4f155903c055b 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java
@@ -25,23 +25,29 @@ public class StructuredPropertiesMapper {
   public static final StructuredPropertiesMapper INSTANCE = new StructuredPropertiesMapper();
 
   public static com.linkedin.datahub.graphql.generated.StructuredProperties map(
-      @Nullable QueryContext context, @Nonnull final StructuredProperties structuredProperties) {
-    return INSTANCE.apply(context, structuredProperties);
+      @Nullable QueryContext context,
+      @Nonnull final StructuredProperties structuredProperties,
+      @Nonnull final Urn entityUrn) {
+    return INSTANCE.apply(context, structuredProperties, entityUrn);
   }
 
   public com.linkedin.datahub.graphql.generated.StructuredProperties apply(
-      @Nullable QueryContext context, @Nonnull final StructuredProperties structuredProperties) {
+      @Nullable QueryContext context,
+      @Nonnull final StructuredProperties structuredProperties,
+      @Nonnull final Urn entityUrn) {
     com.linkedin.datahub.graphql.generated.StructuredProperties result =
         new com.linkedin.datahub.graphql.generated.StructuredProperties();
     result.setProperties(
         structuredProperties.getProperties().stream()
-            .map(p -> mapStructuredProperty(context, p))
+            .map(p -> mapStructuredProperty(context, p, entityUrn))
             .collect(Collectors.toList()));
     return result;
   }
 
   private StructuredPropertiesEntry mapStructuredProperty(
-      @Nullable QueryContext context, StructuredPropertyValueAssignment valueAssignment) {
+      @Nullable QueryContext context,
+      StructuredPropertyValueAssignment valueAssignment,
+      @Nonnull final Urn entityUrn) {
     StructuredPropertiesEntry entry = new StructuredPropertiesEntry();
     entry.setStructuredProperty(createStructuredPropertyEntity(valueAssignment));
     final List<PropertyValue> values = new ArrayList<>();
@@ -58,6 +64,7 @@ private StructuredPropertiesEntry mapStructuredProperty(
             });
     entry.setValues(values);
     entry.setValueEntities(entities);
+    entry.setAssociatedUrn(entityUrn.toString());
     return entry;
   }
 
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java
index c539c65118ac6..5dc73d9ad0938 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java
@@ -17,6 +17,7 @@
 import com.linkedin.datahub.graphql.generated.StringValue;
 import com.linkedin.datahub.graphql.generated.StructuredPropertyDefinition;
 import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity;
+import com.linkedin.datahub.graphql.generated.StructuredPropertySettings;
 import com.linkedin.datahub.graphql.generated.TypeQualifier;
 import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
 import com.linkedin.datahub.graphql.types.mappers.MapperUtils;
@@ -55,6 +56,8 @@ public StructuredPropertyEntity apply(
     MappingHelper<StructuredPropertyEntity> mappingHelper = new MappingHelper<>(aspectMap, result);
     mappingHelper.mapToResult(
         STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, (this::mapStructuredPropertyDefinition));
+    mappingHelper.mapToResult(
+        STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, (this::mapStructuredPropertySettings));
     return mappingHelper.getResult();
   }
 
@@ -112,6 +115,21 @@ private List<AllowedValue> mapAllowedValues(@Nonnull PropertyValueArray gmsValue
     return allowedValues;
   }
 
+  private void mapStructuredPropertySettings(
+      @Nonnull StructuredPropertyEntity extendedProperty, @Nonnull DataMap dataMap) {
+    com.linkedin.structured.StructuredPropertySettings gmsSettings =
+        new com.linkedin.structured.StructuredPropertySettings(dataMap);
+    StructuredPropertySettings settings = new StructuredPropertySettings();
+
+    settings.setIsHidden(gmsSettings.isIsHidden());
+    settings.setShowInSearchFilters(gmsSettings.isShowInSearchFilters());
+    settings.setShowInAssetSummary(gmsSettings.isShowInAssetSummary());
+    settings.setShowAsAssetBadge(gmsSettings.isShowAsAssetBadge());
+    settings.setShowInColumnsTable(gmsSettings.isShowInColumnsTable());
+
+    extendedProperty.setSettings(settings);
+  }
+
   private DataTypeEntity createDataTypeEntity(final Urn dataTypeUrn) {
     final DataTypeEntity dataType = new DataTypeEntity();
     dataType.setUrn(dataTypeUrn.toString());
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java
index 22e161d320f21..e451e96a3e84d 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java
@@ -27,7 +27,8 @@ public class StructuredPropertyType
     implements com.linkedin.datahub.graphql.types.EntityType<StructuredPropertyEntity, String> {
 
   public static final Set<String> ASPECTS_TO_FETCH =
-      ImmutableSet.of(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME);
+      ImmutableSet.of(
+          STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME);
   private final EntityClient _entityClient;
 
   @Override
diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql
index 262d2384d84ad..2868890368723 100644
--- a/datahub-graphql-core/src/main/resources/app.graphql
+++ b/datahub-graphql-core/src/main/resources/app.graphql
@@ -156,6 +156,15 @@ type PlatformPrivileges {
   """
   manageBusinessAttributes: Boolean!
 
+  """
+  Whether the user can create, edit, and delete structured properties.
+  """
+  manageStructuredProperties: Boolean!
+
+  """
+  Whether the user can view the manage structured properties page.
+  """
+  viewStructuredPropertiesPage: Boolean!
 }
 
 """
@@ -517,6 +526,11 @@ type FeatureFlagsConfig {
   If turned on, all siblings will be separated with no way to get to a "combined" sibling view
   """
   showSeparateSiblings: Boolean!
+
+  """
+  If turned on, show the manage structured properties tab in the govern dropdown
+  """
+  showManageStructuredProperties: Boolean!
 }
 
 """
diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
index 16ef59114f86c..9abf4e16f12dd 100644
--- a/datahub-graphql-core/src/main/resources/entity.graphql
+++ b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -262,8 +262,16 @@ type Query {
     Fetch all Business Attributes
     """
     listBusinessAttributes(input: ListBusinessAttributesInput!): ListBusinessAttributesResult
+
+    """
+    Fetch a Data Process Instance by primary key (urn)
+    """
+    dataProcessInstance(urn: String!): DataProcessInstance
+
+
 }
 
+
 """
 An ERModelRelationship is a high-level abstraction that dictates what datasets fields are erModelRelationshiped.
 """
@@ -645,7 +653,7 @@ type Mutation {
     updateERModelRelationship(
         "The urn of the ERModelRelationship to update"
         urn: String!,
-        "Input required to updat an existing DataHub View"
+        "Input required to update an existing DataHub View"
         input: ERModelRelationshipUpdateInput!): Boolean
 
     """
@@ -795,7 +803,7 @@ type Mutation {
     updateView(
       "The urn of the View to update"
       urn: String!,
-      "Input required to updat an existing DataHub View"
+      "Input required to update an existing DataHub View"
       input: UpdateViewInput!): DataHubView
 
     """
@@ -2837,6 +2845,11 @@ type Container implements Entity {
     """
     exists: Boolean
 
+    """
+	The Roles and the properties to access the container
+	"""
+	access: Access
+
     """
     Experimental API.
     For fetching extra entities that do not have custom UI code yet
@@ -3000,8 +3013,14 @@ type InstitutionalMemoryMetadata {
 
     """
     The author of this metadata
+    Deprecated! Use actor instead for users or groups.
     """
-    author: CorpUser!
+    author: CorpUser! @deprecated(reason: "Use `actor`")
+
+    """
+    The author of this metadata
+    """
+    actor: ResolvedActor!
 
     """
     An AuditStamp corresponding to the creation of this resource
@@ -3827,8 +3846,15 @@ enum CorpUserStatus {
   A User that has been provisioned and logged in
   """
   ACTIVE
+
+  """
+  A user that has been suspended
+  """
+  SUSPENDED
 }
 
+union ResolvedActor = CorpUser | CorpGroup
+
 """
 A DataHub User entity, which represents a Person on the Metadata Entity Graph
 """
@@ -9814,13 +9840,45 @@ type MLModelGroup implements EntityWithRelationships & Entity & BrowsableEntity
     privileges: EntityPrivileges
 }
 
+"""
+Properties describing a group of related ML models
+"""
 type MLModelGroupProperties {
+    """
+    Display name of the model group
+    """
+    name: String
 
-description: String
+    """
+    Detailed description of the model group's purpose and contents
+    """
+    description: String
 
-    createdAt: Long
+    """
+    When this model group was created
+    """
+    created: AuditStamp
+
+    """
+    When this model group was last modified
+    """
+    lastModified: AuditStamp
 
+    """
+    Version identifier for this model group
+    """
     version: VersionTag
+
+    """
+    Custom key-value properties for the model group
+    """
+    customProperties: [CustomPropertiesEntry!]
+
+    """
+    Deprecated creation timestamp
+    @deprecated Use the 'created' field instead
+    """
+    createdAt: Long @deprecated(reason: "Use `created` instead")
 }
 
 """
@@ -9970,40 +10028,103 @@ description: String
 }
 
 type MLMetric {
+    """
+    Name of the metric (e.g. accuracy, precision, recall)
+    """
     name: String
 
+    """
+    Description of what this metric measures
+    """
     description: String
 
+    """
+    The computed value of the metric
+    """
     value: String
 
+    """
+    Timestamp when this metric was recorded
+    """
     createdAt: Long
 }
 
 type MLModelProperties {
+    """
+    The display name of the model used in the UI
+    """
+    name: String!
 
+    """
+    Detailed description of the model's purpose and characteristics
+    """
     description: String
 
-    date: Long
+    """
+    When the model was last modified
+    """
+    lastModified: AuditStamp
 
+    """
+    Version identifier for this model
+    """
     version: String
 
+    """
+    The type/category of ML model (e.g. classification, regression)
+    """
     type: String
 
+    """
+    Mapping of hyperparameter configurations
+    """
     hyperParameters: HyperParameterMap
 
-    hyperParams:  [MLHyperParam]
+    """
+    List of hyperparameter settings used to train this model
+    """
+    hyperParams: [MLHyperParam]
 
+    """
+    Performance metrics from model training
+    """
     trainingMetrics: [MLMetric]
 
+    """
+    Names of ML features used by this model
+    """
     mlFeatures: [String!]
 
+    """
+    Tags for categorizing and searching models
+    """
     tags: [String!]
 
+    """
+    Model groups this model belongs to
+    """
     groups: [MLModelGroup]
 
+    """
+    Additional custom properties specific to this model
+    """
     customProperties: [CustomPropertiesEntry!]
 
+    """
+    URL to view this model in external system
+    """
     externalUrl: String
+
+    """
+    When this model was created
+    """
+    created: AuditStamp
+
+    """
+    Deprecated timestamp for model creation
+    @deprecated Use 'created' field instead
+    """
+    date: Long @deprecated(reason: "Use `created` instead")
 }
 
 type MLFeatureProperties {
@@ -10015,6 +10136,8 @@ type MLFeatureProperties {
     version: VersionTag
 
     sources: [Dataset]
+
+    customProperties: [CustomPropertiesEntry!]
 }
 
 """
@@ -10151,13 +10274,15 @@ type MLPrimaryKey implements EntityWithRelationships & Entity {
 
 type MLPrimaryKeyProperties {
 
-description: String
+    description: String
 
     dataType: MLFeatureDataType
 
     version: VersionTag
 
     sources: [Dataset]
+
+    customProperties: [CustomPropertiesEntry!]
 }
 
 """
@@ -10334,7 +10459,7 @@ type MLModelGroupEditableProperties {
 
 type MLFeatureTableProperties {
 
-description: String
+    description: String
 
     mlFeatures: [MLFeature]
 
@@ -12780,3 +12905,92 @@ type CronSchedule {
     """
     timezone: String!
 }
+
+
+"""
+Properties describing a data process instance's execution metadata
+"""
+type DataProcessInstanceProperties {
+    """
+    The display name of this process instance
+    """
+    name: String!
+
+    """
+    URL to view this process instance in the external system
+    """
+    externalUrl: String
+
+    """
+    When this process instance was created
+    """
+    created: AuditStamp
+
+    """
+    Additional custom properties specific to this process instance
+    """
+    customProperties: [CustomPropertiesEntry!]
+}
+
+"""
+Properties specific to an ML model training run instance
+"""
+type MLTrainingRunProperties {
+    """
+    Unique identifier for this training run
+    """
+    id: String
+
+    """
+    List of URLs to access training run outputs (e.g. model artifacts, logs)
+    """
+    outputUrls: [String]
+    
+    """
+    Hyperparameters used in this training run
+    """
+    hyperParams: [MLHyperParam]
+
+    """
+    Performance metrics recorded during this training run
+    """
+    trainingMetrics: [MLMetric]
+}
+
+extend type DataProcessInstance {
+
+    """
+    Additional read only properties associated with the Data Job
+    """
+    properties: DataProcessInstanceProperties
+
+    """
+    The specific instance of the data platform that this entity belongs to
+    """
+    dataPlatformInstance: DataPlatformInstance
+
+    """
+    Sub Types that this entity implements
+    """
+    subTypes: SubTypes
+
+    """
+    The parent container in which the entity resides
+    """
+    container: Container
+
+    """
+    Standardized platform urn where the data process instance is defined
+    """
+    platform: DataPlatform!
+
+    """
+    Recursively get the lineage of containers for this entity
+    """
+    parentContainers: ParentContainersResult
+
+    """
+    Additional properties when subtype is Training Run
+    """
+    mlTrainingRunProperties: MLTrainingRunProperties
+}
diff --git a/datahub-graphql-core/src/main/resources/ingestion.graphql b/datahub-graphql-core/src/main/resources/ingestion.graphql
index 77327ae6d4db1..719ffea30c3dd 100644
--- a/datahub-graphql-core/src/main/resources/ingestion.graphql
+++ b/datahub-graphql-core/src/main/resources/ingestion.graphql
@@ -448,6 +448,11 @@ input ListIngestionSourcesInput {
   Optional Facet filters to apply to the result set
   """
   filters: [FacetFilterInput!]
+
+  """
+  Optional sort order. Defaults to use systemCreated.
+  """
+  sort: SortCriterion
 }
 
 """
diff --git a/datahub-graphql-core/src/main/resources/properties.graphql b/datahub-graphql-core/src/main/resources/properties.graphql
index 292381d064f36..ff20caa50bf03 100644
--- a/datahub-graphql-core/src/main/resources/properties.graphql
+++ b/datahub-graphql-core/src/main/resources/properties.graphql
@@ -49,6 +49,11 @@ type StructuredPropertyEntity implements Entity {
     """
     definition: StructuredPropertyDefinition!
 
+    """
+    Definition of this structured property including its name
+    """
+    settings: StructuredPropertySettings
+
     """
     Granular API for querying edges extending from this entity
     """
@@ -117,6 +122,36 @@ type StructuredPropertyDefinition {
     lastModified: ResolvedAuditStamp
 }
 
+"""
+Settings specific to a structured property entity
+"""
+type StructuredPropertySettings {
+    """
+    Whether or not this asset should be hidden in the main application
+    """
+    isHidden: Boolean!
+
+    """
+    Whether or not this asset should be displayed as a search filter
+    """
+    showInSearchFilters: Boolean!
+
+    """
+    Whether or not this asset should be displayed in the asset sidebar
+    """
+    showInAssetSummary: Boolean!
+
+    """
+    Whether or not this asset should be displayed as an asset badge on other asset's headers
+    """
+    showAsAssetBadge: Boolean!
+
+    """
+    Whether or not this asset should be displayed as a column in the schema field table in a Dataset's "Columns" tab.
+    """
+    showInColumnsTable: Boolean!
+}
+
 """
 An entry for an allowed value for a structured property
 """
@@ -202,6 +237,11 @@ type StructuredPropertiesEntry {
     The optional entities associated with the values if the values are entity urns
     """
     valueEntities: [Entity]
+
+    """
+    The urn of the entity this property came from for tracking purposes e.g. when sibling nodes are merged together
+    """
+    associatedUrn: String!
 }
 
 """
@@ -330,8 +370,9 @@ input CreateStructuredPropertyInput {
 
     """
     The unique fully qualified name of this structured property, dot delimited.
+    This will be required to match the ID of this structured property.
     """
-    qualifiedName: String!
+    qualifiedName: String
 
     """
     The optional display name for this property
@@ -375,6 +416,11 @@ input CreateStructuredPropertyInput {
     For example: ["urn:li:entityType:datahub.dataset"]
     """
     entityTypes: [String!]!
+
+    """
+    Settings for this structured property
+    """
+    settings: StructuredPropertySettingsInput
 }
 
 """
@@ -455,6 +501,11 @@ input UpdateStructuredPropertyInput {
     For backwards compatibility, this is append only.
     """
     newEntityTypes: [String!]
+
+    """
+    Settings for this structured property
+    """
+    settings: StructuredPropertySettingsInput
 }
 
 """
@@ -477,3 +528,34 @@ input DeleteStructuredPropertyInput {
     """
     urn: String!
 }
+
+"""
+Settings for a structured property
+"""
+input StructuredPropertySettingsInput {
+    """
+    Whether or not this asset should be hidden in the main application
+    """
+    isHidden: Boolean
+
+    """
+    Whether or not this asset should be displayed as a search filter
+    """
+    showInSearchFilters: Boolean
+
+    """
+    Whether or not this asset should be displayed in the asset sidebar
+    """
+    showInAssetSummary: Boolean
+
+    """
+    Whether or not this asset should be displayed as an asset badge on other asset's headers
+    """
+    showAsAssetBadge: Boolean
+
+    """
+    Whether or not this asset should be displayed as a column in the schema field table in a Dataset's "Columns" tab.
+    """
+    showInColumnsTable: Boolean
+}
+
diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql
index d0f669f05f959..82bfb9ee26fc4 100644
--- a/datahub-graphql-core/src/main/resources/search.graphql
+++ b/datahub-graphql-core/src/main/resources/search.graphql
@@ -167,6 +167,11 @@ input  SearchFlags {
   fields to include for custom Highlighting
   """
   customHighlightingFields: [String!]
+
+  """
+  Whether or not to fetch and request for structured property facets when doing a search
+  """
+  includeStructuredPropertyFacets: Boolean
 }
 
 """
@@ -872,6 +877,11 @@ type FacetMetadata {
   """
   displayName: String
 
+  """
+  Entity corresponding to the facet
+  """
+  entity: Entity
+
   """
   Aggregated search result counts by value of the field
   """
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java
index 05428788dc3c9..dc22255b1537c 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java
@@ -28,7 +28,7 @@
 public class ListIngestionSourceResolverTest {
 
   private static final ListIngestionSourcesInput TEST_INPUT =
-      new ListIngestionSourcesInput(0, 20, null, null);
+      new ListIngestionSourcesInput(0, 20, null, null, null);
 
   @Test
   public void testGetSuccess() throws Exception {
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java
index 1b33118bd154a..0a8e4e8b4fa5f 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java
@@ -386,7 +386,11 @@ private static EntityClient initMockEntityClient(
     Mockito.when(
             client.searchAcrossEntities(
                 any(),
-                Mockito.eq(entityTypes),
+                Mockito.argThat(
+                    argument ->
+                        argument != null
+                            && argument.containsAll(entityTypes)
+                            && entityTypes.containsAll(argument)),
                 Mockito.eq(query),
                 Mockito.eq(filter),
                 Mockito.eq(start),
@@ -409,7 +413,11 @@ private static void verifyMockEntityClient(
     Mockito.verify(mockClient, Mockito.times(1))
         .searchAcrossEntities(
             any(),
-            Mockito.eq(entityTypes),
+            Mockito.argThat(
+                argument ->
+                    argument != null
+                        && argument.containsAll(entityTypes)
+                        && entityTypes.containsAll(argument)),
             Mockito.eq(query),
             Mockito.eq(filter),
             Mockito.eq(start),
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java
index a601a815453b2..89d218683e33e 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java
@@ -462,12 +462,17 @@ private static EntityClient initMockEntityClient(
     Mockito.when(
             client.searchAcrossEntities(
                 any(),
-                Mockito.eq(entityTypes),
+                Mockito.argThat(
+                    argument ->
+                        argument != null
+                            && argument.containsAll(entityTypes)
+                            && entityTypes.containsAll(argument)),
                 Mockito.eq(query),
                 Mockito.eq(filter),
                 Mockito.eq(start),
                 Mockito.eq(limit),
-                Mockito.eq(Collections.emptyList())))
+                Mockito.eq(Collections.emptyList()),
+                Mockito.eq(null)))
         .thenReturn(result);
     return client;
   }
@@ -483,12 +488,17 @@ private static void verifyMockEntityClient(
     Mockito.verify(mockClient, Mockito.times(1))
         .searchAcrossEntities(
             any(),
-            Mockito.eq(entityTypes),
+            Mockito.argThat(
+                argument ->
+                    argument != null
+                        && argument.containsAll(entityTypes)
+                        && entityTypes.containsAll(argument)),
             Mockito.eq(query),
             Mockito.eq(filter),
             Mockito.eq(start),
             Mockito.eq(limit),
-            Mockito.eq(Collections.emptyList()));
+            Mockito.eq(Collections.emptyList()),
+            Mockito.eq(null));
   }
 
   private static void verifyMockViewService(ViewService mockService, Urn viewUrn) {
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java
index 72cdb78542e41..fec2251f92b63 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java
@@ -10,11 +10,11 @@
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput;
 import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity;
+import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
-import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.r2.RemoteInvocationException;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.ArrayList;
@@ -36,7 +36,8 @@ public class CreateStructuredPropertyResolverTest {
           null,
           null,
           null,
-          new ArrayList<>());
+          new ArrayList<>(),
+          null);
 
   @Test
   public void testGetSuccess() throws Exception {
@@ -56,7 +57,40 @@ public void testGetSuccess() throws Exception {
 
     // Validate that we called ingest
     Mockito.verify(mockEntityClient, Mockito.times(1))
-        .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false));
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
+  }
+
+  @Test
+  public void testGetMismatchIdAndQualifiedName() throws Exception {
+    EntityClient mockEntityClient = initMockEntityClient(true);
+    CreateStructuredPropertyResolver resolver =
+        new CreateStructuredPropertyResolver(mockEntityClient);
+
+    CreateStructuredPropertyInput testInput =
+        new CreateStructuredPropertyInput(
+            "mismatched",
+            "io.acryl.test",
+            "Display Name",
+            "description",
+            true,
+            null,
+            null,
+            null,
+            null,
+            new ArrayList<>(),
+            null);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join());
+
+    // Validate ingest is not called
+    Mockito.verify(mockEntityClient, Mockito.times(0))
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
   }
 
   @Test
@@ -75,7 +109,7 @@ public void testGetUnauthorized() throws Exception {
 
     // Validate that we did NOT call ingest
     Mockito.verify(mockEntityClient, Mockito.times(0))
-        .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false));
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
   }
 
   @Test
@@ -94,7 +128,83 @@ public void testGetFailure() throws Exception {
 
     // Validate that ingest was called, but that caused a failure
     Mockito.verify(mockEntityClient, Mockito.times(1))
-        .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false));
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
+  }
+
+  @Test
+  public void testGetInvalidSettingsInput() throws Exception {
+    EntityClient mockEntityClient = initMockEntityClient(true);
+    CreateStructuredPropertyResolver resolver =
+        new CreateStructuredPropertyResolver(mockEntityClient);
+
+    // if isHidden is true, other fields should not be true
+    StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput();
+    settingsInput.setIsHidden(true);
+    settingsInput.setShowAsAssetBadge(true);
+
+    CreateStructuredPropertyInput testInput =
+        new CreateStructuredPropertyInput(
+            null,
+            "io.acryl.test",
+            "Display Name",
+            "description",
+            true,
+            null,
+            null,
+            null,
+            null,
+            new ArrayList<>(),
+            settingsInput);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join());
+
+    // Validate ingest is not called
+    Mockito.verify(mockEntityClient, Mockito.times(0))
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
+  }
+
+  @Test
+  public void testGetSuccessWithSettings() throws Exception {
+    EntityClient mockEntityClient = initMockEntityClient(true);
+    CreateStructuredPropertyResolver resolver =
+        new CreateStructuredPropertyResolver(mockEntityClient);
+
+    StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput();
+    settingsInput.setShowAsAssetBadge(true);
+
+    CreateStructuredPropertyInput testInput =
+        new CreateStructuredPropertyInput(
+            null,
+            "io.acryl.test",
+            "Display Name",
+            "description",
+            true,
+            null,
+            null,
+            null,
+            null,
+            new ArrayList<>(),
+            settingsInput);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    StructuredPropertyEntity prop = resolver.get(mockEnv).get();
+
+    assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN);
+
+    // Validate that we called ingest
+    Mockito.verify(mockEntityClient, Mockito.times(1))
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
   }
 
   private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception {
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java
new file mode 100644
index 0000000000000..7ecec25708f2d
--- /dev/null
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java
@@ -0,0 +1,91 @@
+package com.linkedin.datahub.graphql.resolvers.structuredproperties;
+
+import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext;
+import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext;
+import static org.mockito.ArgumentMatchers.any;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput;
+import com.linkedin.entity.client.EntityClient;
+import com.linkedin.r2.RemoteInvocationException;
+import graphql.schema.DataFetchingEnvironment;
+import java.util.concurrent.CompletionException;
+import org.mockito.Mockito;
+import org.testng.annotations.Test;
+
+public class DeleteStructuredPropertyResolverTest {
+  private static final String TEST_PROP_URN = "urn:li:structuredProperty:test";
+
+  private static final DeleteStructuredPropertyInput TEST_INPUT =
+      new DeleteStructuredPropertyInput(TEST_PROP_URN);
+
+  @Test
+  public void testGetSuccess() throws Exception {
+    EntityClient mockEntityClient = initMockEntityClient(true);
+    DeleteStructuredPropertyResolver resolver =
+        new DeleteStructuredPropertyResolver(mockEntityClient);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    Boolean success = resolver.get(mockEnv).get();
+    assertTrue(success);
+
+    // Validate that we called delete
+    Mockito.verify(mockEntityClient, Mockito.times(1))
+        .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN)));
+  }
+
+  @Test
+  public void testGetUnauthorized() throws Exception {
+    EntityClient mockEntityClient = initMockEntityClient(true);
+    DeleteStructuredPropertyResolver resolver =
+        new DeleteStructuredPropertyResolver(mockEntityClient);
+
+    // Execute resolver
+    QueryContext mockContext = getMockDenyContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join());
+
+    // Validate that we did NOT call delete
+    Mockito.verify(mockEntityClient, Mockito.times(0))
+        .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN)));
+  }
+
+  @Test
+  public void testGetFailure() throws Exception {
+    EntityClient mockEntityClient = initMockEntityClient(false);
+    DeleteStructuredPropertyResolver resolver =
+        new DeleteStructuredPropertyResolver(mockEntityClient);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join());
+
+    // Validate that deleteEntity was called, but since it's the thing that failed it was called
+    // once still
+    Mockito.verify(mockEntityClient, Mockito.times(1))
+        .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN)));
+  }
+
+  private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception {
+    EntityClient client = Mockito.mock(EntityClient.class);
+    if (!shouldSucceed) {
+      Mockito.doThrow(new RemoteInvocationException()).when(client).deleteEntity(any(), any());
+    }
+    return client;
+  }
+}
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java
new file mode 100644
index 0000000000000..0e9d064b3c7af
--- /dev/null
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java
@@ -0,0 +1,42 @@
+package com.linkedin.datahub.graphql.resolvers.structuredproperties;
+
+import static org.testng.Assert.*;
+
+import com.linkedin.metadata.models.StructuredPropertyUtils;
+import java.util.UUID;
+import org.testng.annotations.Test;
+
+public class StructuredPropertyUtilsTest {
+
+  @Test
+  public void testGetIdMismatchedInput() throws Exception {
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> StructuredPropertyUtils.getPropertyId("test1", "test2"));
+  }
+
+  @Test
+  public void testGetIdConsistentInput() throws Exception {
+    assertEquals(StructuredPropertyUtils.getPropertyId("test1", "test1"), "test1");
+  }
+
+  @Test
+  public void testGetIdNullQualifiedName() throws Exception {
+    assertEquals(StructuredPropertyUtils.getPropertyId("test1", null), "test1");
+  }
+
+  @Test
+  public void testGetIdNullId() throws Exception {
+    assertEquals(StructuredPropertyUtils.getPropertyId(null, "test1"), "test1");
+  }
+
+  @Test
+  public void testGetIdNullForBoth() throws Exception {
+    try {
+      String id = StructuredPropertyUtils.getPropertyId(null, null);
+      UUID.fromString(id);
+    } catch (Exception e) {
+      fail("ID produced is not a UUID");
+    }
+  }
+}
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java
index b818bcfb7d7f4..2b0e7fd83b7ce 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java
@@ -2,20 +2,25 @@
 
 import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext;
 import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext;
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
 import static org.mockito.ArgumentMatchers.any;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertThrows;
 
+import com.linkedin.common.UrnArray;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity;
+import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput;
 import com.linkedin.datahub.graphql.generated.UpdateStructuredPropertyInput;
+import com.linkedin.entity.Aspect;
 import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
-import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.r2.RemoteInvocationException;
+import com.linkedin.structured.StructuredPropertyDefinition;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.concurrent.CompletionException;
 import org.mockito.Mockito;
@@ -33,6 +38,7 @@ public class UpdateStructuredPropertyResolverTest {
           null,
           null,
           null,
+          null,
           null);
 
   @Test
@@ -53,7 +59,7 @@ public void testGetSuccess() throws Exception {
 
     // Validate that we called ingest
     Mockito.verify(mockEntityClient, Mockito.times(1))
-        .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false));
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
   }
 
   @Test
@@ -72,7 +78,7 @@ public void testGetUnauthorized() throws Exception {
 
     // Validate that we did NOT call ingest
     Mockito.verify(mockEntityClient, Mockito.times(0))
-        .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false));
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
   }
 
   @Test
@@ -91,7 +97,80 @@ public void testGetFailure() throws Exception {
 
     // Validate that ingest was not called since there was a get failure before ingesting
     Mockito.verify(mockEntityClient, Mockito.times(0))
-        .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false));
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
+  }
+
+  @Test
+  public void testGetInvalidSettingsInput() throws Exception {
+    EntityClient mockEntityClient = initMockEntityClient(true);
+    UpdateStructuredPropertyResolver resolver =
+        new UpdateStructuredPropertyResolver(mockEntityClient);
+
+    // if isHidden is true, other fields should not be true
+    StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput();
+    settingsInput.setIsHidden(true);
+    settingsInput.setShowInSearchFilters(true);
+
+    final UpdateStructuredPropertyInput testInput =
+        new UpdateStructuredPropertyInput(
+            TEST_STRUCTURED_PROPERTY_URN,
+            "New Display Name",
+            "new description",
+            true,
+            null,
+            null,
+            null,
+            null,
+            settingsInput);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join());
+
+    // Validate that ingest was not called since there was a get failure before ingesting
+    Mockito.verify(mockEntityClient, Mockito.times(0))
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
+  }
+
+  @Test
+  public void testGetValidSettingsInput() throws Exception {
+    EntityClient mockEntityClient = initMockEntityClient(true);
+    UpdateStructuredPropertyResolver resolver =
+        new UpdateStructuredPropertyResolver(mockEntityClient);
+
+    // if isHidden is true, other fields should not be true
+    StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput();
+    settingsInput.setIsHidden(true);
+
+    final UpdateStructuredPropertyInput testInput =
+        new UpdateStructuredPropertyInput(
+            TEST_STRUCTURED_PROPERTY_URN,
+            "New Display Name",
+            "new description",
+            true,
+            null,
+            null,
+            null,
+            null,
+            settingsInput);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    StructuredPropertyEntity prop = resolver.get(mockEnv).get();
+
+    assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN);
+
+    // Validate that we called ingest
+    Mockito.verify(mockEntityClient, Mockito.times(1))
+        .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false));
   }
 
   private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception {
@@ -99,7 +178,11 @@ private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exceptio
     EntityResponse response = new EntityResponse();
     response.setEntityName(Constants.STRUCTURED_PROPERTY_ENTITY_NAME);
     response.setUrn(UrnUtils.getUrn(TEST_STRUCTURED_PROPERTY_URN));
-    response.setAspects(new EnvelopedAspectMap());
+    final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();
+    aspectMap.put(
+        STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+        new EnvelopedAspect().setValue(new Aspect(createDefinition().data())));
+    response.setAspects(aspectMap);
     if (shouldSucceed) {
       Mockito.when(
               client.getV2(
@@ -120,4 +203,13 @@ private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exceptio
 
     return client;
   }
+
+  private StructuredPropertyDefinition createDefinition() {
+    StructuredPropertyDefinition definition = new StructuredPropertyDefinition();
+    definition.setDisplayName("test");
+    definition.setQualifiedName("test");
+    definition.setValueType(UrnUtils.getUrn("urn:li:dataType:datahub.string"));
+    definition.setEntityTypes(new UrnArray());
+    return definition;
+  }
 }
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java
new file mode 100644
index 0000000000000..ab1140d238031
--- /dev/null
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java
@@ -0,0 +1,108 @@
+package com.linkedin.datahub.graphql.utils;
+
+import static org.mockito.ArgumentMatchers.anySet;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.DataMap;
+import com.linkedin.datahub.graphql.analytics.service.AnalyticsUtil;
+import com.linkedin.datahub.graphql.generated.Cell;
+import com.linkedin.datahub.graphql.generated.Row;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.entity.EnvelopedAspectMap;
+import com.linkedin.entity.client.EntityClient;
+import com.linkedin.identity.CorpUserEditableInfo;
+import com.linkedin.identity.CorpUserInfo;
+import com.linkedin.metadata.Constants;
+import io.datahubproject.metadata.context.OperationContext;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class AnalyticsUtilTest {
+
+  @Mock private OperationContext mockOpContext;
+
+  @Mock private EntityClient mockEntityClient;
+
+  final String TEST_CORP_USER_INFO_TEST_USER = "Corp User";
+  final String TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE = "Editable Info Title";
+  final String TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL = "Editable Info Email";
+
+  @BeforeMethod
+  public void setUp() {
+    MockitoAnnotations.openMocks(this);
+  }
+
+  @Test
+  public void testConvertToUserInfoRows() throws Exception {
+    List<Row> rows = new ArrayList<>();
+    rows.add(new Row(null, Arrays.asList(new Cell("urn:li:corpuser:testuser", null, null))));
+
+    // create a CorpUserInfo with only display name set
+    CorpUserInfo corpUserInfo = new CorpUserInfo();
+    corpUserInfo.setActive(true);
+    corpUserInfo.setDisplayName(TEST_CORP_USER_INFO_TEST_USER);
+
+    // create an editableInfo with the email and title set
+    CorpUserEditableInfo corpUserEditableInfo = new CorpUserEditableInfo();
+    corpUserEditableInfo.setEmail(TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL); // Overriding email
+    corpUserEditableInfo.setTitle(TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE); // Overriding title
+
+    DataMap corpUserInfoDataMap = new DataMap();
+    corpUserInfoDataMap.put("name", Constants.CORP_USER_INFO_ASPECT_NAME);
+    corpUserInfoDataMap.put("type", "VERSIONED");
+    corpUserInfoDataMap.put("value", corpUserInfo.data());
+
+    DataMap corpUserEditableInfoDataMap = new DataMap();
+    corpUserEditableInfoDataMap.put("name", Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME);
+    corpUserEditableInfoDataMap.put("type", "VERSIONED");
+    corpUserEditableInfoDataMap.put("value", corpUserEditableInfo.data());
+
+    EnvelopedAspect corpUserInfoEnvelopedAspect = new EnvelopedAspect(corpUserInfoDataMap);
+    EnvelopedAspect corpUserEditableInfoEnvelopedAspect =
+        new EnvelopedAspect(corpUserEditableInfoDataMap);
+
+    EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();
+    aspectMap.put(Constants.CORP_USER_INFO_ASPECT_NAME, corpUserInfoEnvelopedAspect);
+    aspectMap.put(
+        Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME, corpUserEditableInfoEnvelopedAspect);
+
+    EntityResponse entityResponse = new EntityResponse();
+    entityResponse.setAspects(aspectMap);
+
+    Map<Urn, EntityResponse> entityResponseMap = new HashMap<>();
+    Urn userUrn = UrnUtils.getUrn("urn:li:corpuser:testuser");
+    entityResponseMap.put(userUrn, entityResponse);
+
+    // method of the entity client we need to mock to retrieve the response map
+    when(mockEntityClient.batchGetV2(
+            eq(mockOpContext), eq(Constants.CORP_USER_ENTITY_NAME), anySet(), anySet()))
+        .thenReturn(entityResponseMap);
+
+    // function we are testing
+    AnalyticsUtil.convertToUserInfoRows(mockOpContext, mockEntityClient, rows);
+
+    Row updatedRow = rows.get(0);
+    List<Cell> updatedCells = updatedRow.getCells();
+
+    // asserting that the display user is from CorpUserInfo and email, title are from EditableInfo
+    assertEquals(updatedCells.get(0).getValue(), TEST_CORP_USER_INFO_TEST_USER);
+    assertEquals(
+        updatedCells.get(1).getValue(),
+        TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE); // Overriding title
+    assertEquals(
+        updatedCells.get(2).getValue(),
+        TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL); // Overriding email
+  }
+}
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java
index 6ecbc8d015b29..4383df9d46a4b 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java
@@ -47,4 +47,26 @@ public void testStartOfNextWeek() {
     Mockito.when(dateUtil.getNow()).thenReturn(setTimeParts(8, false));
     assertEqualStartOfNextWeek(dateUtil, 9);
   }
+
+  // validates logic to display correct dates in MAU chart
+  @Test
+  public void testDateAdjustmentsForMonth() {
+    DateUtil dateUtil = Mockito.spy(DateUtil.class);
+
+    Mockito.when(dateUtil.getNow()).thenReturn(new DateTime(2024, 11, 15, 0, 0, 0));
+
+    // start date should be next month minus a day
+    // but we want to display Dec 1 instead of Nov 30, so add a day and verify it's Dec
+    DateTime startOfNextMonthMinus12 = dateUtil.getStartOfNextMonth().minusMonths(12);
+    DateTime adjustedStart = startOfNextMonthMinus12.minusMillis(1).plusDays(1);
+    assertEquals(12, adjustedStart.getMonthOfYear()); // Verify it is December
+    assertEquals(2023, adjustedStart.getYear()); // Verify it is 2023
+
+    // verify that the end date displays correctly
+    // the chart will display Oct 1 as the last month because we don't show current month
+    DateTime startOfThisMonth = dateUtil.getStartOfThisMonth();
+    DateTime adjustedEnd = startOfThisMonth.minusMillis(1).plusDays(1);
+    assertEquals(11, adjustedEnd.getMonthOfYear()); // Verify it is November
+    assertEquals(2024, adjustedEnd.getYear()); // Verify it is 2024
+  }
 }
diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
index b783efa09713d..a3b2e9ad6b3e2 100644
--- a/datahub-upgrade/build.gradle
+++ b/datahub-upgrade/build.gradle
@@ -5,6 +5,7 @@ plugins {
 }
 
 apply from: "../gradle/versioning/versioning.gradle"
+apply from: "../gradle/coverage/java-coverage.gradle"
 
 ext {
   docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry
@@ -59,7 +60,7 @@ dependencies {
   // mock internal schema registry
   implementation externalDependency.kafkaAvroSerde
   implementation externalDependency.kafkaAvroSerializer
-  implementation "org.apache.kafka:kafka_2.12:3.7.1"
+  implementation "org.apache.kafka:kafka_2.13:3.7.2"
 
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillIngestionSourceInfoIndicesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillIngestionSourceInfoIndicesConfig.java
new file mode 100644
index 0000000000000..f525c4e35875d
--- /dev/null
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillIngestionSourceInfoIndicesConfig.java
@@ -0,0 +1,29 @@
+package com.linkedin.datahub.upgrade.config;
+
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.datahub.upgrade.system.ingestion.BackfillIngestionSourceInfoIndices;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class)
+public class BackfillIngestionSourceInfoIndicesConfig {
+
+  @Bean
+  public NonBlockingSystemUpgrade backfillIngestionSourceInfoIndices(
+      final OperationContext opContext,
+      final EntityService<?> entityService,
+      final AspectDao aspectDao,
+      @Value("${systemUpdate.ingestionIndices.enabled}") final boolean enabled,
+      @Value("${systemUpdate.ingestionIndices.batchSize}") final Integer batchSize,
+      @Value("${systemUpdate.ingestionIndices.delayMs}") final Integer delayMs,
+      @Value("${systemUpdate.ingestionIndices.limit}") final Integer limit) {
+    return new BackfillIngestionSourceInfoIndices(
+        opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit);
+  }
+}
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java
index f3a4c47c59f0b..fdd84da6044f7 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java
@@ -13,6 +13,7 @@
 import com.linkedin.gms.factory.kafka.common.TopicConventionFactory;
 import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory;
 import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.dao.producer.KafkaEventProducer;
 import com.linkedin.metadata.dao.producer.KafkaHealthChecker;
@@ -29,6 +30,7 @@
 import io.datahubproject.metadata.context.OperationContextConfig;
 import io.datahubproject.metadata.context.RetrieverContext;
 import io.datahubproject.metadata.context.ServicesRegistryContext;
+import io.datahubproject.metadata.context.ValidationContext;
 import io.datahubproject.metadata.services.RestrictedService;
 import java.util.List;
 import javax.annotation.Nonnull;
@@ -161,7 +163,8 @@ protected OperationContext javaSystemOperationContext(
       @Nonnull final GraphService graphService,
       @Nonnull final SearchService searchService,
       @Qualifier("baseElasticSearchComponents")
-          BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components) {
+          BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components,
+      @Nonnull final ConfigurationProvider configurationProvider) {
 
     EntityServiceAspectRetriever entityServiceAspectRetriever =
         EntityServiceAspectRetriever.builder()
@@ -184,8 +187,13 @@ protected OperationContext javaSystemOperationContext(
             components.getIndexConvention(),
             RetrieverContext.builder()
                 .aspectRetriever(entityServiceAspectRetriever)
+                .cachingAspectRetriever(CachingAspectRetriever.EMPTY)
                 .graphRetriever(systemGraphRetriever)
                 .searchRetriever(searchServiceSearchRetriever)
+                .build(),
+            ValidationContext.builder()
+                .alternateValidation(
+                    configurationProvider.getFeatureFlags().isAlternateMCPValidation())
                 .build());
 
     entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext);
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/PropertyDefinitionsConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/PropertyDefinitionsConfig.java
new file mode 100644
index 0000000000000..49cd0beb98ea2
--- /dev/null
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/PropertyDefinitionsConfig.java
@@ -0,0 +1,30 @@
+package com.linkedin.datahub.upgrade.config.restoreindices;
+
+import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.datahub.upgrade.system.restoreindices.structuredproperties.PropertyDefinitions;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class)
+public class PropertyDefinitionsConfig {
+
+  @Bean
+  public NonBlockingSystemUpgrade propertyDefinitions(
+      final OperationContext opContext,
+      final EntityService<?> entityService,
+      final AspectDao aspectDao,
+      @Value("${systemUpdate.propertyDefinitions.enabled}") final boolean enabled,
+      @Value("${systemUpdate.propertyDefinitions.batchSize}") final Integer batchSize,
+      @Value("${systemUpdate.propertyDefinitions.delayMs}") final Integer delayMs,
+      @Value("${systemUpdate.propertyDefinitions.limit}") final Integer limit) {
+    return new PropertyDefinitions(
+        opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit);
+  }
+}
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDomainDescriptionConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexDomainDescriptionConfig.java
similarity index 84%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDomainDescriptionConfig.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexDomainDescriptionConfig.java
index 3cdab0dc4d4bc..0ec9eb38e2b1c 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDomainDescriptionConfig.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexDomainDescriptionConfig.java
@@ -1,7 +1,8 @@
-package com.linkedin.datahub.upgrade.config;
+package com.linkedin.datahub.upgrade.config.restoreindices;
 
+import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
-import com.linkedin.datahub.upgrade.system.domaindescription.ReindexDomainDescription;
+import com.linkedin.datahub.upgrade.system.restoreindices.domaindescription.ReindexDomainDescription;
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import io.datahubproject.metadata.context.OperationContext;
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/graph/ReindexDataJobViaNodesCLLConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/graph/ReindexDataJobViaNodesCLLConfig.java
similarity index 88%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/graph/ReindexDataJobViaNodesCLLConfig.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/graph/ReindexDataJobViaNodesCLLConfig.java
index a973876c6715f..730991cd0a697 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/graph/ReindexDataJobViaNodesCLLConfig.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/graph/ReindexDataJobViaNodesCLLConfig.java
@@ -1,8 +1,8 @@
-package com.linkedin.datahub.upgrade.config.graph;
+package com.linkedin.datahub.upgrade.config.restoreindices.graph;
 
 import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
-import com.linkedin.datahub.upgrade.system.graph.vianodes.ReindexDataJobViaNodesCLL;
+import com.linkedin.datahub.upgrade.system.restoreindices.graph.vianodes.ReindexDataJobViaNodesCLL;
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import io.datahubproject.metadata.context.OperationContext;
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/graph/ReindexEdgeStatusConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/graph/ReindexEdgeStatusConfig.java
similarity index 89%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/graph/ReindexEdgeStatusConfig.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/graph/ReindexEdgeStatusConfig.java
index 97715573eb51f..14b60f44c09a0 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/graph/ReindexEdgeStatusConfig.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/graph/ReindexEdgeStatusConfig.java
@@ -1,8 +1,8 @@
-package com.linkedin.datahub.upgrade.config.graph;
+package com.linkedin.datahub.upgrade.config.restoreindices.graph;
 
 import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
-import com.linkedin.datahub.upgrade.system.graph.edgestatus.ReindexEdgeStatus;
+import com.linkedin.datahub.upgrade.system.restoreindices.graph.edgestatus.ReindexEdgeStatus;
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import io.datahubproject.metadata.context.OperationContext;
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java
index 4d53b603c1eaf..1e5cd6cdb2417 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java
@@ -180,7 +180,7 @@ private void readerExecutable(ReaderWrapper reader, UpgradeContext context) {
       try {
         aspectRecord =
             EntityUtils.toSystemAspect(
-                    context.opContext().getRetrieverContext().get(), aspect.toEntityAspect())
+                    context.opContext().getRetrieverContext(), aspect.toEntityAspect())
                 .get()
                 .getRecordTemplate();
       } catch (Exception e) {
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java
index 6c70aee88675c..56feffd211bcd 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java
@@ -1,13 +1,12 @@
 package com.linkedin.datahub.upgrade.system;
 
-import static com.linkedin.metadata.Constants.DATA_HUB_UPGRADE_RESULT_ASPECT_NAME;
-
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.upgrade.UpgradeContext;
 import com.linkedin.datahub.upgrade.UpgradeStep;
 import com.linkedin.datahub.upgrade.UpgradeStepResult;
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
 import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.aspect.SystemAspect;
 import com.linkedin.metadata.boot.BootstrapStep;
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
@@ -16,10 +15,13 @@
 import com.linkedin.metadata.entity.ebean.PartitionedStream;
 import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs;
 import com.linkedin.metadata.utils.AuditStampUtils;
+import com.linkedin.upgrade.DataHubUpgradeResult;
 import com.linkedin.upgrade.DataHubUpgradeState;
 import com.linkedin.util.Pair;
 import io.datahubproject.metadata.context.OperationContext;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.function.Function;
@@ -33,6 +35,8 @@
  */
 @Slf4j
 public abstract class AbstractMCLStep implements UpgradeStep {
+  public static final String LAST_URN_KEY = "lastUrn";
+
   private final OperationContext opContext;
   private final EntityService<?> entityService;
   private final AspectDao aspectDao;
@@ -70,10 +74,30 @@ protected Urn getUpgradeIdUrn() {
   @Override
   public Function<UpgradeContext, UpgradeStepResult> executable() {
     return (context) -> {
+      // Resume state
+      Optional<DataHubUpgradeResult> prevResult =
+          context.upgrade().getUpgradeResult(opContext, getUpgradeIdUrn(), entityService);
+      String resumeUrn =
+          prevResult
+              .filter(
+                  result ->
+                      DataHubUpgradeState.IN_PROGRESS.equals(result.getState())
+                          && result.getResult() != null
+                          && result.getResult().containsKey(LAST_URN_KEY))
+              .map(result -> result.getResult().get(LAST_URN_KEY))
+              .orElse(null);
+      if (resumeUrn != null) {
+        log.info("{}: Resuming from URN: {}", getUpgradeIdUrn(), resumeUrn);
+      }
 
       // re-using for configuring the sql scan
       RestoreIndicesArgs args =
-          new RestoreIndicesArgs().aspectName(getAspectName()).batchSize(batchSize).limit(limit);
+          new RestoreIndicesArgs()
+              .aspectName(getAspectName())
+              .batchSize(batchSize)
+              .lastUrn(resumeUrn)
+              .urnBasedPagination(resumeUrn != null)
+              .limit(limit);
 
       if (getUrnLike() != null) {
         args = args.urnLike(getUrnLike());
@@ -86,40 +110,61 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
                 batch -> {
                   log.info("Processing batch({}) of size {}.", getAspectName(), batchSize);
 
-                  List<Pair<Future<?>, Boolean>> futures;
-
+                  List<Pair<Future<?>, SystemAspect>> futures;
                   futures =
                       EntityUtils.toSystemAspectFromEbeanAspects(
-                              opContext.getRetrieverContext().get(),
-                              batch.collect(Collectors.toList()))
+                              opContext.getRetrieverContext(), batch.collect(Collectors.toList()))
                           .stream()
                           .map(
-                              systemAspect ->
-                                  entityService.alwaysProduceMCLAsync(
-                                      opContext,
-                                      systemAspect.getUrn(),
-                                      systemAspect.getUrn().getEntityType(),
-                                      getAspectName(),
-                                      systemAspect.getAspectSpec(),
-                                      null,
-                                      systemAspect.getRecordTemplate(),
-                                      null,
-                                      systemAspect
-                                          .getSystemMetadata()
-                                          .setRunId(id())
-                                          .setLastObserved(System.currentTimeMillis()),
-                                      AuditStampUtils.createDefaultAuditStamp(),
-                                      ChangeType.UPSERT))
-                          .collect(Collectors.toList());
-
-                  futures.forEach(
-                      f -> {
-                        try {
-                          f.getFirst().get();
-                        } catch (InterruptedException | ExecutionException e) {
-                          throw new RuntimeException(e);
-                        }
-                      });
+                              systemAspect -> {
+                                Pair<Future<?>, Boolean> future =
+                                    entityService.alwaysProduceMCLAsync(
+                                        opContext,
+                                        systemAspect.getUrn(),
+                                        systemAspect.getUrn().getEntityType(),
+                                        getAspectName(),
+                                        systemAspect.getAspectSpec(),
+                                        null,
+                                        systemAspect.getRecordTemplate(),
+                                        null,
+                                        systemAspect
+                                            .getSystemMetadata()
+                                            .setRunId(id())
+                                            .setLastObserved(System.currentTimeMillis()),
+                                        AuditStampUtils.createDefaultAuditStamp(),
+                                        ChangeType.UPSERT);
+                                return Pair.<Future<?>, SystemAspect>of(
+                                    future.getFirst(), systemAspect);
+                              })
+                          .toList();
+
+                  SystemAspect lastAspect =
+                      futures.stream()
+                          .map(
+                              f -> {
+                                try {
+                                  f.getFirst().get();
+                                  return f.getSecond();
+                                } catch (InterruptedException | ExecutionException e) {
+                                  throw new RuntimeException(e);
+                                }
+                              })
+                          .reduce((a, b) -> b)
+                          .orElse(null);
+
+                  // record progress
+                  if (lastAspect != null) {
+                    log.info(
+                        "{}: Saving state. Last urn:{}", getUpgradeIdUrn(), lastAspect.getUrn());
+                    context
+                        .upgrade()
+                        .setUpgradeResult(
+                            opContext,
+                            getUpgradeIdUrn(),
+                            entityService,
+                            DataHubUpgradeState.IN_PROGRESS,
+                            Map.of(LAST_URN_KEY, lastAspect.getUrn().toString()));
+                  }
 
                   if (batchDelayMs > 0) {
                     log.info("Sleeping for {} ms", batchDelayMs);
@@ -142,12 +187,23 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
   @Override
   /** Returns whether the upgrade should be skipped. */
   public boolean skip(UpgradeContext context) {
-    boolean previouslyRun =
-        entityService.exists(
-            opContext, getUpgradeIdUrn(), DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, true);
-    if (previouslyRun) {
-      log.info("{} was already run. Skipping.", id());
+    Optional<DataHubUpgradeResult> prevResult =
+        context.upgrade().getUpgradeResult(opContext, getUpgradeIdUrn(), entityService);
+
+    boolean previousRunFinal =
+        prevResult
+            .filter(
+                result ->
+                    DataHubUpgradeState.SUCCEEDED.equals(result.getState())
+                        || DataHubUpgradeState.ABORTED.equals(result.getState()))
+            .isPresent();
+
+    if (previousRunFinal) {
+      log.info(
+          "{} was already run. State: {} Skipping.",
+          id(),
+          prevResult.map(DataHubUpgradeResult::getState));
     }
-    return previouslyRun;
+    return previousRunFinal;
   }
 }
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java
index b8b7e828c16c6..5b807c6c450af 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java
@@ -31,6 +31,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
 import org.springframework.core.io.ClassPathResource;
@@ -53,6 +54,7 @@ static List<UpgradeStep> generateSteps(
             .getBootstrap()
             .getTemplates()
             .stream()
+            .map(cfg -> cfg.withOverride(opContext.getObjectMapper()))
             .filter(cfg -> cfg.isBlocking() == isBlocking)
             .map(cfg -> new BootstrapMCPStep(opContext, entityService, cfg))
             .collect(Collectors.toList());
@@ -98,8 +100,8 @@ static AspectsBatch generateAspectBatch(
             .collect(Collectors.toList());
 
     return AspectsBatchImpl.builder()
-        .mcps(mcps, auditStamp, opContext.getRetrieverContext().get())
-        .retrieverContext(opContext.getRetrieverContext().get())
+        .mcps(mcps, auditStamp, opContext.getRetrieverContext())
+        .retrieverContext(opContext.getRetrieverContext())
         .build();
   }
 
@@ -109,13 +111,29 @@ static List<ObjectNode> resolveMCPTemplate(
       AuditStamp auditStamp)
       throws IOException {
 
-    String template = loadTemplate(mcpTemplate.getMcps_location());
-    Mustache mustache = MUSTACHE_FACTORY.compile(new StringReader(template), mcpTemplate.getName());
+    final String template = loadTemplate(mcpTemplate.getMcps_location());
     Map<String, Object> scopeValues = resolveValues(opContext, mcpTemplate, auditStamp);
+
     StringWriter writer = new StringWriter();
-    mustache.execute(writer, scopeValues);
+    try {
+      Mustache mustache =
+          MUSTACHE_FACTORY.compile(new StringReader(template), mcpTemplate.getName());
+      mustache.execute(writer, scopeValues);
+    } catch (Exception e) {
+      log.error(
+          "Failed to apply mustache template. Template: {} Values: {}",
+          template,
+          resolveEnv(mcpTemplate));
+      throw e;
+    }
 
-    return opContext.getYamlMapper().readValue(writer.toString(), new TypeReference<>() {});
+    final String yaml = writer.toString();
+    try {
+      return opContext.getYamlMapper().readValue(yaml, new TypeReference<>() {});
+    } catch (Exception e) {
+      log.error("Failed to parse rendered MCP bootstrap yaml: {}", yaml);
+      throw e;
+    }
   }
 
   static Map<String, Object> resolveValues(
@@ -128,13 +146,21 @@ static Map<String, Object> resolveValues(
     // built-in
     scopeValues.put("auditStamp", RecordUtils.toJsonString(auditStamp));
 
+    String envValue = resolveEnv(mcpTemplate);
+    if (envValue != null) {
+      scopeValues.putAll(opContext.getObjectMapper().readValue(envValue, new TypeReference<>() {}));
+    }
+    return scopeValues;
+  }
+
+  @Nullable
+  private static String resolveEnv(BootstrapMCPConfigFile.MCPTemplate mcpTemplate) {
     if (mcpTemplate.getValues_env() != null
         && !mcpTemplate.getValues_env().isEmpty()
         && System.getenv().containsKey(mcpTemplate.getValues_env())) {
-      String envValue = System.getenv(mcpTemplate.getValues_env());
-      scopeValues.putAll(opContext.getObjectMapper().readValue(envValue, new TypeReference<>() {}));
+      return System.getenv(mcpTemplate.getValues_env());
     }
-    return scopeValues;
+    return null;
   }
 
   private static String loadTemplate(String source) throws IOException {
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java
index 8fd3dd7c7d897..009d19e453b6a 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java
@@ -1,5 +1,7 @@
 package com.linkedin.datahub.upgrade.system.bootstrapmcps.model;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
 import java.util.List;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -7,6 +9,7 @@
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
 @AllArgsConstructor
 @NoArgsConstructor
@@ -23,6 +26,7 @@ public static class Bootstrap {
     private List<MCPTemplate> templates;
   }
 
+  @Slf4j
   @AllArgsConstructor
   @NoArgsConstructor
   @Data
@@ -36,5 +40,19 @@ public static class MCPTemplate {
     @Builder.Default private boolean optional = false;
     @Nonnull private String mcps_location;
     @Nullable private String values_env;
+    @Nullable private String revision_env;
+
+    public MCPTemplate withOverride(ObjectMapper objectMapper) {
+      if (revision_env != null) {
+        String overrideJson = System.getenv().getOrDefault(revision_env, "{}");
+        try {
+          return objectMapper.readerForUpdating(this).readValue(overrideJson);
+        } catch (IOException e) {
+          log.error("Error applying override {} to {}", overrideJson, this);
+          throw new RuntimeException(e);
+        }
+      }
+      return this;
+    }
   }
 }
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java
index 55cdcae931ab5..1bdea10123999 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java
@@ -2,6 +2,8 @@
 
 import static com.linkedin.metadata.Constants.*;
 
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Throwables;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.upgrade.UpgradeContext;
@@ -23,8 +25,6 @@
 import java.util.Set;
 import java.util.function.Function;
 import lombok.extern.slf4j.Slf4j;
-import org.codehaus.jackson.node.JsonNodeFactory;
-import org.codehaus.jackson.node.ObjectNode;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.RequestOptions;
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndices.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndices.java
new file mode 100644
index 0000000000000..70f0844367f67
--- /dev/null
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndices.java
@@ -0,0 +1,43 @@
+package com.linkedin.datahub.upgrade.system.ingestion;
+
+import com.google.common.collect.ImmutableList;
+import com.linkedin.datahub.upgrade.UpgradeStep;
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import java.util.List;
+import javax.annotation.Nonnull;
+
+public class BackfillIngestionSourceInfoIndices implements NonBlockingSystemUpgrade {
+
+  private final List<UpgradeStep> _steps;
+
+  public BackfillIngestionSourceInfoIndices(
+      @Nonnull OperationContext opContext,
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      boolean enabled,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
+    if (enabled) {
+      _steps =
+          ImmutableList.of(
+              new BackfillIngestionSourceInfoIndicesStep(
+                  opContext, entityService, aspectDao, batchSize, batchDelayMs, limit));
+    } else {
+      _steps = ImmutableList.of();
+    }
+  }
+
+  @Override
+  public String id() {
+    return getClass().getSimpleName();
+  }
+
+  @Override
+  public List<UpgradeStep> steps() {
+    return _steps;
+  }
+}
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndicesStep.java
new file mode 100644
index 0000000000000..2525a57bfd7ec
--- /dev/null
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndicesStep.java
@@ -0,0 +1,56 @@
+package com.linkedin.datahub.upgrade.system.ingestion;
+
+import static com.linkedin.metadata.Constants.*;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.upgrade.system.AbstractMCLStep;
+import com.linkedin.metadata.boot.BootstrapStep;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class BackfillIngestionSourceInfoIndicesStep extends AbstractMCLStep {
+
+  private static final String UPGRADE_ID = BackfillIngestionSourceInfoIndices.class.getSimpleName();
+  private static final Urn UPGRADE_ID_URN = BootstrapStep.getUpgradeUrn(UPGRADE_ID);
+
+  public BackfillIngestionSourceInfoIndicesStep(
+      @Nonnull OperationContext opContext,
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
+    super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit);
+  }
+
+  @Override
+  public String id() {
+    return UPGRADE_ID;
+  }
+
+  @Nonnull
+  @Override
+  protected String getAspectName() {
+    return INGESTION_INFO_ASPECT_NAME;
+  }
+
+  @Nullable
+  @Override
+  protected String getUrnLike() {
+    return "urn:li:" + INGESTION_SOURCE_ENTITY_NAME + ":%";
+  }
+
+  /**
+   * Returns whether the upgrade should proceed if the step fails after exceeding the maximum
+   * retries.
+   */
+  @Override
+  public boolean isOptional() {
+    return true;
+  }
+}
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescription.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/domaindescription/ReindexDomainDescription.java
similarity index 94%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescription.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/domaindescription/ReindexDomainDescription.java
index 85af912e24f68..6eb7e83135627 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescription.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/domaindescription/ReindexDomainDescription.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.domaindescription;
+package com.linkedin.datahub.upgrade.system.restoreindices.domaindescription;
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.upgrade.UpgradeStep;
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescriptionStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/domaindescription/ReindexDomainDescriptionStep.java
similarity index 93%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescriptionStep.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/domaindescription/ReindexDomainDescriptionStep.java
index 1fa8bc92af078..c452300e7f61c 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescriptionStep.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/domaindescription/ReindexDomainDescriptionStep.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.domaindescription;
+package com.linkedin.datahub.upgrade.system.restoreindices.domaindescription;
 
 import static com.linkedin.metadata.Constants.*;
 
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/edgestatus/ReindexEdgeStatus.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/edgestatus/ReindexEdgeStatus.java
similarity index 94%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/edgestatus/ReindexEdgeStatus.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/edgestatus/ReindexEdgeStatus.java
index 6b7286a6a0639..7296d0880a863 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/edgestatus/ReindexEdgeStatus.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/edgestatus/ReindexEdgeStatus.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.graph.edgestatus;
+package com.linkedin.datahub.upgrade.system.restoreindices.graph.edgestatus;
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.upgrade.UpgradeStep;
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/edgestatus/ReindexReindexEdgeStatusStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/edgestatus/ReindexReindexEdgeStatusStep.java
similarity index 95%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/edgestatus/ReindexReindexEdgeStatusStep.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/edgestatus/ReindexReindexEdgeStatusStep.java
index 6543f82e74563..955442ab85379 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/edgestatus/ReindexReindexEdgeStatusStep.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/edgestatus/ReindexReindexEdgeStatusStep.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.graph.edgestatus;
+package com.linkedin.datahub.upgrade.system.restoreindices.graph.edgestatus;
 
 import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME;
 
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/vianodes/ReindexDataJobViaNodesCLL.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/vianodes/ReindexDataJobViaNodesCLL.java
similarity index 94%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/vianodes/ReindexDataJobViaNodesCLL.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/vianodes/ReindexDataJobViaNodesCLL.java
index 7a4ca9586f155..75bfcabdb82d3 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/vianodes/ReindexDataJobViaNodesCLL.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/vianodes/ReindexDataJobViaNodesCLL.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.graph.vianodes;
+package com.linkedin.datahub.upgrade.system.restoreindices.graph.vianodes;
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.upgrade.UpgradeStep;
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/vianodes/ReindexDataJobViaNodesCLLStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/vianodes/ReindexDataJobViaNodesCLLStep.java
similarity index 95%
rename from datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/vianodes/ReindexDataJobViaNodesCLLStep.java
rename to datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/vianodes/ReindexDataJobViaNodesCLLStep.java
index e3e07f99bb1ee..4087e35c0e96b 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/graph/vianodes/ReindexDataJobViaNodesCLLStep.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/graph/vianodes/ReindexDataJobViaNodesCLLStep.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.graph.vianodes;
+package com.linkedin.datahub.upgrade.system.restoreindices.graph.vianodes;
 
 import static com.linkedin.metadata.Constants.*;
 
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitions.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitions.java
new file mode 100644
index 0000000000000..2c3a26cdc43dc
--- /dev/null
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitions.java
@@ -0,0 +1,49 @@
+package com.linkedin.datahub.upgrade.system.restoreindices.structuredproperties;
+
+import com.google.common.collect.ImmutableList;
+import com.linkedin.datahub.upgrade.UpgradeStep;
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import java.util.List;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * A job that reindexes all domain aspects as part of reindexing descriptions This is required to
+ * fix the analytics for domains
+ */
+@Slf4j
+public class PropertyDefinitions implements NonBlockingSystemUpgrade {
+
+  private final List<UpgradeStep> _steps;
+
+  public PropertyDefinitions(
+      @Nonnull OperationContext opContext,
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      boolean enabled,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
+    if (enabled) {
+      _steps =
+          ImmutableList.of(
+              new PropertyDefinitionsStep(
+                  opContext, entityService, aspectDao, batchSize, batchDelayMs, limit));
+    } else {
+      _steps = ImmutableList.of();
+    }
+  }
+
+  @Override
+  public String id() {
+    return this.getClass().getName();
+  }
+
+  @Override
+  public List<UpgradeStep> steps() {
+    return _steps;
+  }
+}
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitionsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitionsStep.java
new file mode 100644
index 0000000000000..1f5e979a163af
--- /dev/null
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitionsStep.java
@@ -0,0 +1,42 @@
+package com.linkedin.datahub.upgrade.system.restoreindices.structuredproperties;
+
+import static com.linkedin.metadata.Constants.*;
+
+import com.linkedin.datahub.upgrade.system.AbstractMCLStep;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.Nullable;
+
+@Slf4j
+public class PropertyDefinitionsStep extends AbstractMCLStep {
+
+  public PropertyDefinitionsStep(
+      OperationContext opContext,
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
+    super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit);
+  }
+
+  @Override
+  public String id() {
+    return "structured-property-definitions-v1";
+  }
+
+  @Nonnull
+  @Override
+  protected String getAspectName() {
+    return STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
+  }
+
+  @Nullable
+  @Override
+  protected String getUrnLike() {
+    return "urn:li:" + STRUCTURED_PROPERTY_ENTITY_NAME + ":%";
+  }
+}
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java
index eece83f4ab713..de03538907432 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java
@@ -1,5 +1,6 @@
 package com.linkedin.datahub.upgrade.system.schemafield;
 
+import static com.linkedin.datahub.upgrade.system.AbstractMCLStep.LAST_URN_KEY;
 import static com.linkedin.metadata.Constants.APP_SOURCE;
 import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.SCHEMA_METADATA_ASPECT_NAME;
@@ -61,7 +62,6 @@
  */
 @Slf4j
 public class GenerateSchemaFieldsFromSchemaMetadataStep implements UpgradeStep {
-  private static final String LAST_URN_KEY = "lastUrn";
   private static final List<String> REQUIRED_ASPECTS =
       List.of(SCHEMA_METADATA_ASPECT_NAME, STATUS_ASPECT_NAME);
 
@@ -168,13 +168,13 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
 
                   AspectsBatch aspectsBatch =
                       AspectsBatchImpl.builder()
-                          .retrieverContext(opContext.getRetrieverContext().get())
+                          .retrieverContext(opContext.getRetrieverContext())
                           .items(
                               batch
                                   .flatMap(
                                       ebeanAspectV2 ->
                                           EntityUtils.toSystemAspectFromEbeanAspects(
-                                              opContext.getRetrieverContext().get(),
+                                              opContext.getRetrieverContext(),
                                               Set.of(ebeanAspectV2))
                                               .stream())
                                   .map(
@@ -189,11 +189,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
                                               .auditStamp(systemAspect.getAuditStamp())
                                               .systemMetadata(
                                                   withAppSource(systemAspect.getSystemMetadata()))
-                                              .build(
-                                                  opContext
-                                                      .getRetrieverContext()
-                                                      .get()
-                                                      .getAspectRetriever()))
+                                              .build(opContext.getAspectRetriever()))
                                   .collect(Collectors.toList()))
                           .build();
 
diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java
index 845d818527343..21bc6b725cba2 100644
--- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java
+++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java
@@ -1,18 +1,22 @@
 package com.linkedin.datahub.upgrade;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 import static org.testng.AssertJUnit.assertNotNull;
 
+import com.linkedin.data.template.StringMap;
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager;
 import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking;
 import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCPStep;
-import com.linkedin.datahub.upgrade.system.graph.vianodes.ReindexDataJobViaNodesCLL;
+import com.linkedin.datahub.upgrade.system.restoreindices.graph.vianodes.ReindexDataJobViaNodesCLL;
 import com.linkedin.metadata.boot.kafka.MockSystemUpdateDeserializer;
 import com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
@@ -20,17 +24,30 @@
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.EntityServiceImpl;
+import com.linkedin.metadata.entity.ebean.EbeanAspectV2;
+import com.linkedin.metadata.entity.ebean.PartitionedStream;
 import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs;
 import com.linkedin.mxe.Topics;
+import com.linkedin.upgrade.DataHubUpgradeResult;
+import com.linkedin.upgrade.DataHubUpgradeState;
+import com.linkedin.util.Pair;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
+import java.sql.Timestamp;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.inject.Named;
+import org.mockito.ArgumentCaptor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 @ActiveProfiles("test")
@@ -63,7 +80,12 @@ public class DatahubUpgradeNonBlockingTest extends AbstractTestNGSpringContextTe
 
   @Autowired private EntityServiceImpl entityService;
 
-  @Autowired private OperationContext opContext;
+  private OperationContext opContext;
+
+  @BeforeClass
+  public void init() {
+    opContext = TestOperationContexts.systemContextNoValidate();
+  }
 
   @Test
   public void testSystemUpdateNonBlockingInit() {
@@ -81,10 +103,13 @@ public void testSystemUpdateNonBlockingInit() {
   }
 
   @Test
-  public void testReindexDataJobViaNodesCLLPaging() {
+  public void testReindexDataJobViaNodesCLLPagingArgs() {
     EntityService<?> mockService = mock(EntityService.class);
 
     AspectDao mockAspectDao = mock(AspectDao.class);
+    PartitionedStream<EbeanAspectV2> mockStream = mock(PartitionedStream.class);
+    when(mockStream.partition(anyInt())).thenReturn(Stream.empty());
+    when(mockAspectDao.streamAspectBatches(any(RestoreIndicesArgs.class))).thenReturn(mockStream);
 
     ReindexDataJobViaNodesCLL cllUpgrade =
         new ReindexDataJobViaNodesCLL(opContext, mockService, mockAspectDao, true, 10, 0, 0);
@@ -102,9 +127,79 @@ public void testReindexDataJobViaNodesCLLPaging() {
                     .batchSize(10)
                     .limit(0)
                     .aspectName("dataJobInputOutput")
+                    .urnBasedPagination(false)
+                    .lastUrn(null)
                     .urnLike("urn:li:dataJob:%")));
   }
 
+  @Test
+  public void testReindexDataJobViaNodesCLLResumePaging() throws Exception {
+    // Mock services
+    EntityService<?> mockService = mock(EntityService.class);
+    AspectDao mockAspectDao = mock(AspectDao.class);
+
+    // Create test data
+    EbeanAspectV2 aspect1 = createMockEbeanAspect("urn:li:dataJob:job1", "dataJobInputOutput");
+    EbeanAspectV2 aspect2 = createMockEbeanAspect("urn:li:dataJob:job2", "dataJobInputOutput");
+    EbeanAspectV2 aspect3 = createMockEbeanAspect("urn:li:dataJob:job3", "dataJobInputOutput");
+    List<EbeanAspectV2> initialBatch = Arrays.asList(aspect1, aspect2);
+    List<EbeanAspectV2> resumeBatch = Arrays.asList(aspect3);
+
+    // Mock the stream for first batch
+    PartitionedStream<EbeanAspectV2> initialStream = mock(PartitionedStream.class);
+    when(initialStream.partition(anyInt())).thenReturn(Stream.of(initialBatch.stream()));
+
+    // Mock the stream for second batch
+    PartitionedStream<EbeanAspectV2> resumeStream = mock(PartitionedStream.class);
+    when(resumeStream.partition(anyInt())).thenReturn(Stream.of(resumeBatch.stream()));
+
+    // Setup the AspectDao using Answer to handle null safely
+    when(mockAspectDao.streamAspectBatches(any(RestoreIndicesArgs.class)))
+        .thenAnswer(
+            invocation -> {
+              RestoreIndicesArgs args = invocation.getArgument(0);
+              if (args.lastUrn() == null) {
+                return initialStream;
+              } else if ("urn:li:dataJob:job2".equals(args.lastUrn())) {
+                return resumeStream;
+              }
+              return mock(PartitionedStream.class);
+            });
+
+    // Mock successful MCL production
+    when(mockService.alwaysProduceMCLAsync(
+            any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
+        .thenReturn(Pair.of(CompletableFuture.completedFuture(null), true));
+
+    // Create the upgrade
+    ReindexDataJobViaNodesCLL cllUpgrade =
+        new ReindexDataJobViaNodesCLL(opContext, mockService, mockAspectDao, true, 2, 0, 0);
+
+    // Initial Run
+    cllUpgrade.steps().get(0).executable().apply(createMockInitialUpgrade());
+
+    // Resumed
+    cllUpgrade.steps().get(0).executable().apply(createMockResumeUpgrade());
+
+    // Use ArgumentCaptor to verify the calls
+    ArgumentCaptor<RestoreIndicesArgs> argsCaptor =
+        ArgumentCaptor.forClass(RestoreIndicesArgs.class);
+    verify(mockAspectDao, times(2)).streamAspectBatches(argsCaptor.capture());
+
+    List<RestoreIndicesArgs> capturedArgs = argsCaptor.getAllValues();
+
+    // Verify both the initial and resume calls were made with correct arguments
+    assertEquals(capturedArgs.get(0).lastUrn(), null);
+    assertEquals(capturedArgs.get(0).urnBasedPagination(), false);
+    assertEquals(capturedArgs.get(1).lastUrn(), "urn:li:dataJob:job2");
+    assertEquals(capturedArgs.get(1).urnBasedPagination(), true);
+
+    // Verify MCL production was called for each aspect
+    verify(mockService, times(3))
+        .alwaysProduceMCLAsync(
+            any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any());
+  }
+
   @Test
   public void testNonBlockingBootstrapMCP() {
     List<BootstrapMCPStep> mcpTemplate =
@@ -123,4 +218,54 @@ public void testNonBlockingBootstrapMCP() {
                 .map(update -> update.getMcpTemplate().getName())
                 .collect(Collectors.toSet())));
   }
+
+  private UpgradeContext createMockInitialUpgrade() {
+    // Mock the Upgrade instance
+    Upgrade mockUpgrade = mock(Upgrade.class);
+
+    // Configure the mock upgrade to return no previous result
+    when(mockUpgrade.getUpgradeResult(any(), any(), any())).thenReturn(Optional.empty());
+
+    UpgradeContext mockInitialContext = mock(UpgradeContext.class);
+    when(mockInitialContext.opContext()).thenReturn(opContext);
+    when(mockInitialContext.upgrade()).thenReturn(mockUpgrade);
+    when(mockInitialContext.report()).thenReturn(mock(UpgradeReport.class));
+
+    return mockInitialContext;
+  }
+
+  private UpgradeContext createMockResumeUpgrade() {
+    // Mock the Upgrade instance
+    Upgrade mockUpgrade = mock(Upgrade.class);
+    DataHubUpgradeResult mockPrevResult = mock(DataHubUpgradeResult.class);
+
+    // Configure the mock previous result
+    when(mockPrevResult.getState()).thenReturn(DataHubUpgradeState.IN_PROGRESS);
+    when(mockPrevResult.getResult())
+        .thenReturn(new StringMap(Map.of("lastUrn", "urn:li:dataJob:job2")));
+
+    // Configure the mock upgrade to return our previous result
+    when(mockUpgrade.getUpgradeResult(any(), any(), any())).thenReturn(Optional.of(mockPrevResult));
+
+    UpgradeContext mockResumeContext = mock(UpgradeContext.class);
+    when(mockResumeContext.opContext()).thenReturn(opContext);
+    when(mockResumeContext.upgrade()).thenReturn(mockUpgrade);
+    when(mockResumeContext.report()).thenReturn(mock(UpgradeReport.class));
+
+    return mockResumeContext;
+  }
+
+  private static EbeanAspectV2 createMockEbeanAspect(String urn, String aspectName) {
+    Timestamp now = new Timestamp(System.currentTimeMillis());
+    return new EbeanAspectV2(
+        urn,
+        aspectName,
+        0L,
+        "{}", // metadata
+        now, // createdOn
+        "urn:li:corpuser:testUser", // createdBy
+        null, // createdFor
+        null // systemMetadata
+        );
+  }
 }
diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java
index 81d883d8ce36b..5b7b8756f11fb 100644
--- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java
+++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java
@@ -19,17 +19,17 @@
 @Import(value = {SystemAuthenticationFactory.class})
 public class UpgradeCliApplicationTestConfiguration {
 
-  @MockBean private UpgradeCli upgradeCli;
+  @MockBean public UpgradeCli upgradeCli;
 
-  @MockBean private Database ebeanServer;
+  @MockBean public Database ebeanServer;
 
-  @MockBean private SearchService searchService;
+  @MockBean public SearchService searchService;
 
-  @MockBean private GraphService graphService;
+  @MockBean public GraphService graphService;
 
-  @MockBean private EntityRegistry entityRegistry;
+  @MockBean public EntityRegistry entityRegistry;
 
-  @MockBean ConfigEntityRegistry configEntityRegistry;
+  @MockBean public ConfigEntityRegistry configEntityRegistry;
 
   @MockBean public EntityIndexBuilders entityIndexBuilders;
 
diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java
index 3a2728b4e1d3d..04b1095e770e0 100644
--- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java
+++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java
@@ -22,7 +22,6 @@
 import com.linkedin.upgrade.DataHubUpgradeState;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.RetrieverContext;
-import java.util.Optional;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -48,7 +47,7 @@ public void setup() {
     step =
         new GenerateSchemaFieldsFromSchemaMetadataStep(
             mockOpContext, mockEntityService, mockAspectDao, 10, 100, 1000);
-    when(mockOpContext.getRetrieverContext()).thenReturn(Optional.of(mockRetrieverContext));
+    when(mockOpContext.getRetrieverContext()).thenReturn(mockRetrieverContext);
   }
 
   /** Test to verify the correct step ID is returned. */
diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java
index 68023a084bbd2..f914b355fe780 100644
--- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java
+++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java
@@ -4,6 +4,7 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.UrnUtils;
@@ -17,6 +18,7 @@
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.io.IOException;
 import java.util.List;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Listeners;
 import org.testng.annotations.Test;
 import uk.org.webcompere.systemstubs.environment.EnvironmentVariables;
@@ -28,10 +30,17 @@ public class BootstrapMCPUtilTest {
   static final OperationContext OP_CONTEXT =
       TestOperationContexts.systemContextNoSearchAuthorization();
   private static final String DATAHUB_TEST_VALUES_ENV = "DATAHUB_TEST_VALUES_ENV";
+  private static final String DATAHUB_TEST_REVISION_ENV = "DATAHUB_TEST_REVISION_ENV";
   private static final AuditStamp TEST_AUDIT_STAMP = AuditStampUtils.createDefaultAuditStamp();
 
   @SystemStub private EnvironmentVariables environmentVariables;
 
+  @BeforeMethod
+  private void resetEnvironment() {
+    environmentVariables.remove(DATAHUB_TEST_VALUES_ENV);
+    environmentVariables.remove(DATAHUB_TEST_REVISION_ENV);
+  }
+
   @Test
   public void testResolveYamlConf() throws IOException {
     BootstrapMCPConfigFile initConfig =
@@ -51,9 +60,28 @@ public void testResolveYamlConf() throws IOException {
   }
 
   @Test
-  public void testResolveMCPTemplateDefaults() throws IOException {
-    environmentVariables.remove(DATAHUB_TEST_VALUES_ENV);
+  public void testResolveYamlConfOverride() throws IOException {
+    environmentVariables.set(DATAHUB_TEST_REVISION_ENV, "{\"version\":\"2024110600\"}");
+
+    BootstrapMCPConfigFile initConfig =
+        BootstrapMCPUtil.resolveYamlConf(
+            OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class);
+    assertEquals(initConfig.getBootstrap().getTemplates().size(), 1);
+
+    BootstrapMCPConfigFile.MCPTemplate template =
+        initConfig.getBootstrap().getTemplates().get(0).withOverride(new ObjectMapper());
+    assertEquals(template.getName(), "datahub-test");
+    assertEquals(template.getVersion(), "2024110600");
+    assertFalse(template.isForce());
+    assertFalse(template.isBlocking());
+    assertTrue(template.isAsync());
+    assertFalse(template.isOptional());
+    assertEquals(template.getMcps_location(), "bootstrapmcp/datahub-test-mcp.yaml");
+    assertEquals(template.getValues_env(), "DATAHUB_TEST_VALUES_ENV");
+  }
 
+  @Test
+  public void testResolveMCPTemplateDefaults() throws IOException {
     BootstrapMCPConfigFile.MCPTemplate template =
         BootstrapMCPUtil.resolveYamlConf(
                 OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class)
@@ -186,8 +214,6 @@ public void testResolveMCPTemplateOverride() throws IOException {
 
   @Test
   public void testMCPBatch() throws IOException {
-    environmentVariables.remove(DATAHUB_TEST_VALUES_ENV);
-
     BootstrapMCPConfigFile.MCPTemplate template =
         BootstrapMCPUtil.resolveYamlConf(
                 OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class)
@@ -219,6 +245,6 @@ public void testMCPBatch() throws IOException {
         OP_CONTEXT
             .getObjectMapper()
             .readTree(
-                "{\"source\":{\"type\":\"datahub-gc\",\"config\":{\"cleanup_expired_tokens\":false,\"truncate_indices\":true,\"dataprocess_cleanup\":{\"retention_days\":10,\"delete_empty_data_jobs\":true,\"delete_empty_data_flows\":true,\"hard_delete_entities\":false,\"keep_last_n\":5},\"soft_deleted_entities_cleanup\":{\"retention_days\":10}}}}"));
+                "{\"source\":{\"type\":\"datahub-gc\",\"config\":{\"cleanup_expired_tokens\":false,\"truncate_indices\":true,\"dataprocess_cleanup\":{\"retention_days\":10,\"delete_empty_data_jobs\":true,\"delete_empty_data_flows\":true,\"hard_delete_entities\":false,\"keep_last_n\":5},\"soft_deleted_entities_cleanup\":{\"retention_days\":10},\"execution_request_cleanup\":{\"keep_history_min_count\":10,\"keep_history_max_count\":1000,\"keep_history_max_days\":30,\"batch_read_size\":100,\"enabled\":false}}}}"));
   }
 }
diff --git a/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml b/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml
index d049a807ac1d8..233db06d61c3f 100644
--- a/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml
+++ b/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml
@@ -23,7 +23,13 @@
               keep_last_n: {{dataprocess_cleanup.keep_last_n}}{{^dataprocess_cleanup.keep_last_n}}5{{/dataprocess_cleanup.keep_last_n}}
             soft_deleted_entities_cleanup:
               retention_days: {{soft_deleted_entities_cleanup.retention_days}}{{^soft_deleted_entities_cleanup.retention_days}}10{{/soft_deleted_entities_cleanup.retention_days}}
+            execution_request_cleanup:
+              keep_history_min_count: {{execution_request_cleanup.keep_history_min_count}}{{^execution_request_cleanup.keep_history_min_count}}10{{/execution_request_cleanup.keep_history_min_count}}
+              keep_history_max_count: {{execution_request_cleanup.keep_history_max_count}}{{^execution_request_cleanup.keep_history_max_count}}1000{{/execution_request_cleanup.keep_history_max_count}}
+              keep_history_max_days: {{execution_request_cleanup.keep_history_max_days}}{{^execution_request_cleanup.keep_history_max_days}}30{{/execution_request_cleanup.keep_history_max_days}}
+              batch_read_size: {{execution_request_cleanup.batch_read_size}}{{^execution_request_cleanup.batch_read_size}}100{{/execution_request_cleanup.batch_read_size}}
+              enabled: {{execution_request_cleanup.enabled}}{{^execution_request_cleanup.enabled}}false{{/execution_request_cleanup.enabled}}
       extraArgs: {}
       debugMode: false
       executorId: default
-  headers: {}
\ No newline at end of file
+  headers: {}
diff --git a/datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml b/datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml
index 649cc09632fc2..5718ea3ac0e04 100644
--- a/datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml
+++ b/datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml
@@ -6,4 +6,5 @@ bootstrap:
       # blocking: false
       # async: true
       mcps_location: "bootstrapmcp/datahub-test-mcp.yaml"
-      values_env: "DATAHUB_TEST_VALUES_ENV"
\ No newline at end of file
+      values_env: "DATAHUB_TEST_VALUES_ENV"
+      revision_env: "DATAHUB_TEST_REVISION_ENV"
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/DocTemplate.mdx b/datahub-web-react/.storybook/DocTemplate.mdx
new file mode 100644
index 0000000000000..9ea1250075e11
--- /dev/null
+++ b/datahub-web-react/.storybook/DocTemplate.mdx
@@ -0,0 +1,42 @@
+import React from 'react';
+
+import { ThemeProvider } from 'styled-components';
+import { GlobalStyle } from './styledComponents';
+
+import { Meta, Title, Subtitle, Description, Primary, Controls, Stories } from '@storybook/blocks';
+import { CodeBlock } from '../src/alchemy-components/.docs/mdx-components';
+
+{/* 
+  * 👇 The isTemplate property is required to tell Storybook that this is a template
+  * See https://storybook.js.org/docs/api/doc-block-meta
+  * to learn how to use
+*/}
+
+<Meta isTemplate />
+
+<ThemeProvider theme={{}}>
+	<GlobalStyle />
+
+	<Title />
+
+	<Subtitle />
+
+	<div className="docsDescription">
+		<Description />
+	</div>
+
+	<br />
+
+	### Import
+
+	<CodeBlock />
+
+	<br/>
+
+	### Customize
+
+	<Primary />
+	<Controls />
+
+	<Stories />
+</ThemeProvider>
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/main.js b/datahub-web-react/.storybook/main.js
new file mode 100644
index 0000000000000..2b92dffd88eb3
--- /dev/null
+++ b/datahub-web-react/.storybook/main.js
@@ -0,0 +1,25 @@
+// Docs for badges: https://storybook.js.org/addons/@geometricpanda/storybook-addon-badges
+
+export default {
+	framework: '@storybook/react-vite',
+	features: {
+		buildStoriesJson: true,
+	},
+	core: {
+		disableTelemetry: true,
+	},
+	stories: [
+		'../src/alchemy-components/.docs/*.mdx',
+		'../src/alchemy-components/components/**/*.stories.@(js|jsx|mjs|ts|tsx)'
+	],
+	addons: [
+		'@storybook/addon-onboarding',
+		'@storybook/addon-essentials',
+		'@storybook/addon-interactions',
+		'@storybook/addon-links',
+		'@geometricpanda/storybook-addon-badges',
+	],
+	typescript: {
+		reactDocgen: 'react-docgen-typescript',
+	},
+}
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/manager-head.html b/datahub-web-react/.storybook/manager-head.html
new file mode 100644
index 0000000000000..98e6a2895f45c
--- /dev/null
+++ b/datahub-web-react/.storybook/manager-head.html
@@ -0,0 +1,33 @@
+<style type="text/css">
+	/* Regular */
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 400;
+		src: url('../src/fonts/Mulish-Regular.ttf') format('truetype');
+	}
+
+	/* Medium */
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 500;
+		src: url('../src/fonts/Mulish-Medium.ttf') format('truetype');
+	}
+
+	/* SemiBold */
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 600;
+		src: url('../src/fonts/Mulish-SemiBold.ttf') format('truetype');
+	}
+
+	/* Bold */
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 700;
+		src: url('../src/fonts/Mulish-Bold.ttf') format('truetype');
+	}
+</style>
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/manager.js b/datahub-web-react/.storybook/manager.js
new file mode 100644
index 0000000000000..6e9c62dd96c23
--- /dev/null
+++ b/datahub-web-react/.storybook/manager.js
@@ -0,0 +1,15 @@
+import './storybook-theme.css';
+
+import { addons } from '@storybook/manager-api';
+import acrylTheme from './storybook-theme.js';
+
+// Theme setup
+addons.setConfig({
+	theme: acrylTheme,
+});
+
+// Favicon
+const link = document.createElement('link');
+link.setAttribute('rel', 'shortcut icon');
+link.setAttribute('href', 'https://www.acryldata.io/icons/favicon.ico');
+document.head.appendChild(link);
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/preview-head.html b/datahub-web-react/.storybook/preview-head.html
new file mode 100644
index 0000000000000..98e6a2895f45c
--- /dev/null
+++ b/datahub-web-react/.storybook/preview-head.html
@@ -0,0 +1,33 @@
+<style type="text/css">
+	/* Regular */
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 400;
+		src: url('../src/fonts/Mulish-Regular.ttf') format('truetype');
+	}
+
+	/* Medium */
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 500;
+		src: url('../src/fonts/Mulish-Medium.ttf') format('truetype');
+	}
+
+	/* SemiBold */
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 600;
+		src: url('../src/fonts/Mulish-SemiBold.ttf') format('truetype');
+	}
+
+	/* Bold */
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 700;
+		src: url('../src/fonts/Mulish-Bold.ttf') format('truetype');
+	}
+</style>
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/preview.js b/datahub-web-react/.storybook/preview.js
new file mode 100644
index 0000000000000..a497ce7bccf3c
--- /dev/null
+++ b/datahub-web-react/.storybook/preview.js
@@ -0,0 +1,84 @@
+import './storybook-theme.css';
+// FYI: import of antd styles required to show components based on it correctly
+import 'antd/dist/antd.css';
+
+import { BADGE, defaultBadgesConfig } from '@geometricpanda/storybook-addon-badges';
+import DocTemplate from './DocTemplate.mdx';
+
+const preview = {
+    tags: ['!dev', 'autodocs'],
+    parameters: {
+        previewTabs: {
+            'storybook/docs/panel': { index: -1 },
+        },
+        controls: {
+            matchers: {
+                color: /(background|color)$/i,
+                date: /Date$/i,
+            },
+        },
+        options: {
+            storySort: {
+                method: 'alphabetical',
+                order: [
+                    // Order of Docs Pages
+                    'Introduction',
+                    'Style Guide',
+                    'Design Tokens',
+                    'Style Utilities',
+                    'Icons',
+
+                    // Order of Components
+                    'Layout',
+                    'Forms',
+                    'Data Display',
+                    'Feedback',
+                    'Typography',
+                    'Overlay',
+                    'Disclosure',
+                    'Navigation',
+                    'Media',
+                    'Other',
+                ],
+                locales: '',
+            },
+        },
+        docs: {
+            page: DocTemplate,
+            toc: {
+                disable: false,
+            },
+            docs: {
+                source: {
+                    format: true,
+                },
+            },
+        },
+
+        // Reconfig the premade badges with better titles
+        badgesConfig: {
+            stable: {
+                ...defaultBadgesConfig[BADGE.STABLE],
+                title: 'Stable',
+                tooltip: 'This component is stable but may have frequent changes. Use at own discretion.',
+            },
+            productionReady: {
+                ...defaultBadgesConfig[BADGE.STABLE],
+                title: 'Production Ready',
+                tooltip: 'This component is production ready and has been tested in a production environment.',
+            },
+            WIP: {
+                ...defaultBadgesConfig[BADGE.BETA],
+                title: 'WIP',
+                tooltip: 'This component is a work in progress and may not be fully functional or tested.',
+            },
+            readyForDesignReview: {
+                ...defaultBadgesConfig[BADGE.NEEDS_REVISION],
+                title: 'Ready for Design Review',
+                tooltip: 'This component is ready for design review and feedback.',
+            },
+        },
+    },
+};
+
+export default preview;
diff --git a/datahub-web-react/.storybook/storybook-logo.svg b/datahub-web-react/.storybook/storybook-logo.svg
new file mode 100644
index 0000000000000..5cc86813b5933
--- /dev/null
+++ b/datahub-web-react/.storybook/storybook-logo.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 116.71 125.19"><defs><style>.cls-1{fill:#08303a;}.cls-2{fill:#11696b;}.cls-3{fill:#20d3bd;}</style></defs><g id="artwork"><path class="cls-1" d="M96.39,34.23,79.87,11.08a26.43,26.43,0,0,0-43,0L20.32,34.23A26.42,26.42,0,0,0,41.83,76h33A26.42,26.42,0,0,0,96.39,34.23ZM74.87,68h-33a18.42,18.42,0,0,1-15-29.12L43.35,15.72a18.43,18.43,0,0,1,30,0L89.87,38.88A18.42,18.42,0,0,1,74.87,68Z"/><path class="cls-2" d="M105.89,72.32,73,26.24a18,18,0,0,0-29.31,0L10.82,72.32a18,18,0,0,0,14.65,28.46H91.24a18,18,0,0,0,14.65-28.46ZM91.24,92.78H25.47A10,10,0,0,1,17.33,77L50.21,30.88a10,10,0,0,1,16.28,0L99.38,77A10,10,0,0,1,91.24,92.78Z"/><path class="cls-3" d="M114.83,109.26,66.56,41.61a10.07,10.07,0,0,0-16.41,0L1.88,109.26a10.08,10.08,0,0,0,8.2,15.93h96.55a10.08,10.08,0,0,0,8.2-15.93Zm-8.2,7.93H10.08a2.08,2.08,0,0,1-1.69-3.29L56.66,46.25a2.08,2.08,0,0,1,1.69-.87,2.05,2.05,0,0,1,1.69.87l48.28,67.65A2.08,2.08,0,0,1,106.63,117.19Z"/></g></svg>
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/storybook-theme.css b/datahub-web-react/.storybook/storybook-theme.css
new file mode 100644
index 0000000000000..edf93c57cf208
--- /dev/null
+++ b/datahub-web-react/.storybook/storybook-theme.css
@@ -0,0 +1,263 @@
+/* Storybook Theme CSS Overrides */
+
+/* Regular */
+@font-face {
+	font-family: 'Mulish';
+	font-style: normal;
+	font-weight: 400;
+	src: url('../src/fonts/Mulish-Regular.ttf') format('truetype');
+}
+
+/* Medium */
+@font-face {
+	font-family: 'Mulish';
+	font-style: normal;
+	font-weight: 500;
+	src: url('../src/fonts/Mulish-Medium.ttf') format('truetype');
+}
+
+/* SemiBold */
+@font-face {
+	font-family: 'Mulish';
+	font-style: normal;
+	font-weight: 600;
+	src: url('../src/fonts/Mulish-SemiBold.ttf') format('truetype');
+}
+
+/* Bold */
+@font-face {
+	font-family: 'Mulish';
+	font-style: normal;
+	font-weight: 700;
+	src: url('../src/fonts/Mulish-Bold.ttf') format('truetype');
+}
+
+body {
+	font-family: 'Mulish', sans-serif !important;
+}
+
+::-webkit-scrollbar {
+	height: 8px;
+	width: 8px;
+}
+
+*::-webkit-scrollbar-track {
+	background: rgba(193, 196, 208, 0);
+	border-radius: 10px;
+}
+
+*::-webkit-scrollbar-thumb {
+	background: rgba(193, 196, 208, 0);
+	border-radius: 10px;
+	transition: 0.3s;
+}
+
+*:hover::-webkit-scrollbar-track {
+	background: rgba(193, 196, 208, 0.3);
+}
+
+*:hover::-webkit-scrollbar-thumb {
+	background: rgba(193, 196, 208, 0.8);
+}
+
+.sbdocs-wrapper {
+	max-width: 95% !important;
+}
+
+.sidebar-header img {
+	max-height: 25px !important;
+}
+
+.sb-bar {
+	box-shadow: none !important;
+	border-bottom: 1px solid hsla(203, 50%, 30%, 0.15) !important;
+}
+
+.sbdocs-preview,
+.docblock-argstable-body,
+.docblock-source {
+	box-shadow: none !important;
+	filter: none !important;
+}
+
+.docblock-source {
+	max-width: 100% !important;
+	overflow: auto !important;
+	margin: 1rem 0 !important;
+}
+
+.sidebar-item,
+.sidebar-item[data-selected="true"] {
+	height: 32px !important;
+	display: flex !important;
+	align-items: center !important;
+	padding-right: 0 !important;
+	padding: 6px 12px !important;
+	font-size: 15px !important;
+	margin-bottom: 4px !important;
+	color: #000 !important;
+}
+
+.sidebar-item:hover {
+	background-color: #eff8fc !important;
+}
+
+.sidebar-item>a {
+	align-items: center !important;
+	gap: 8px !important;
+	padding: 0 !important;
+}
+
+.sidebar-item[data-nodetype="group"] {
+	margin-top: 8px !important;
+}
+
+.sidebar-item[data-nodetype="component"] {
+	padding-left: 8px !important;
+}
+
+[data-nodetype="root"]>[data-action="collapse-root"]>div:first-child,
+[data-nodetype="component"] div {
+	display: none !important;
+}
+
+[data-nodetype="document"][data-parent-id],
+[data-nodetype="story"][data-parent-id] {
+	padding: 0 !important;
+	margin-left: 16px !important;
+	height: 18px !important;
+	min-height: auto !important;
+	font-weight: 400 !important;
+}
+
+[data-nodetype="document"][data-parent-id] svg,
+[data-nodetype="story"][data-parent-id] svg {
+	display: none !important;
+}
+
+[data-nodetype="document"][data-parent-id]::before,
+[data-nodetype="story"][data-parent-id]::before {
+	content: '→' !important;
+}
+
+[data-nodetype="document"][data-parent-id]:hover,
+[data-nodetype="story"][data-parent-id]:hover,
+[data-nodetype="document"][data-parent-id][data-selected="true"]:hover,
+[data-nodetype="story"][data-parent-id][data-selected="true"]:hover {
+	background-color: #fff !important;
+	color: #4da1bf !important;
+}
+
+[data-nodetype="document"][data-parent-id][data-selected="true"],
+[data-nodetype="story"][data-parent-id][data-selected="true"] {
+	background-color: #fff !important;
+	height: 18px !important;
+	min-height: auto !important;
+	font-weight: 400 !important;
+}
+
+.sbdocs-content div[id*=--sandbox]~div[id*=--sandbox]~div[id*=--sandbox],
+li:has(a[href="#sandbox"]) {
+	display: none !important;
+}
+
+[data-nodetype="document"]:not([data-parent-id]) {
+	padding-left: 0 !important;
+}
+
+[data-nodetype="document"]:not([data-parent-id]) svg {
+	display: none !important;
+}
+
+[data-nodetype="document"]:not([data-parent-id])>a {
+	font-size: 18px !important;
+	font-weight: 300 !important;
+}
+
+[data-nodetype="component"][aria-expanded="true"],
+[data-nodetype="document"][data-selected="true"] {
+	color: #000 !important;
+	background-color: transparent !important;
+	font-weight: 700 !important;
+}
+
+[data-nodetype="root"][data-selected="true"] {
+	background-color: transparent !important;
+}
+
+[data-nodetype="document"][data-selected="true"],
+[data-nodetype="document"][data-parent-id][data-selected="true"] {
+	color: #4da1bf !important;
+}
+
+.sidebar-subheading {
+	font-size: 12px !important;
+	font-weight: 600 !important;
+	letter-spacing: 1px !important;
+	color: #a9adbd !important;
+}
+
+.sbdocs-wrapper {
+	padding: 2rem !important;
+}
+
+table,
+tr,
+tbody>tr>* {
+	border-color: hsla(203, 50%, 30%, 0.15) !important;
+	background-color: transparent;
+}
+
+:where(table:not(.sb-anchor, .sb-unstyled, .sb-unstyled table)) tr:nth-of-type(2n) {
+	background-color: transparent !important;
+}
+
+tr {
+	border-top: 0 !important;
+}
+
+th {
+	border: 0 !important;
+}
+
+h2#stories {
+	display: none;
+}
+
+.tabbutton {
+	border-bottom: none !important
+}
+
+.tabbutton.tabbutton-active {
+	color: rgb(120, 201, 230) !important;
+}
+
+.toc-wrapper {
+	margin-top: -2.5rem !important;
+	font-family: 'Mulish', sans-serif !important;
+}
+
+/* Custom Doc Styles */
+
+.custom-docs {
+	position: relative;
+}
+
+.acrylBg {
+	position: fixed;
+	bottom: 0;
+	left: -20px;
+	background-repeat: repeat;
+	z-index: 0;
+}
+
+.acrylBg img {
+	filter: invert(8);
+}
+
+.custom-docs p,
+.docsDescription p,
+.custom-docs li {
+	font-size: 16px;
+	line-height: 1.75;
+}
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/storybook-theme.js b/datahub-web-react/.storybook/storybook-theme.js
new file mode 100644
index 0000000000000..462bf2f03da94
--- /dev/null
+++ b/datahub-web-react/.storybook/storybook-theme.js
@@ -0,0 +1,47 @@
+import { create } from '@storybook/theming';
+import brandImage from './storybook-logo.svg';
+
+import theme, { typography } from '../src/alchemy-components/theme';
+
+export default create({
+	// config 
+	base: 'light',
+	brandTitle: 'Acryl Design System',
+	brandUrl: '/?path=/docs/',
+	brandImage: brandImage,
+	brandTarget: '_self',
+
+	// styles 
+	fontBase: typography.fontFamily,
+	fontCode: 'monospace',
+
+	colorPrimary: theme.semanticTokens.colors.primary,
+	colorSecondary: theme.semanticTokens.colors.secondary,
+
+	// UI
+	appBg: theme.semanticTokens.colors['body-bg'],
+	appContentBg: theme.semanticTokens.colors['body-bg'],
+	appPreviewBg: theme.semanticTokens.colors['body-bg'],
+	appBorderColor: theme.semanticTokens.colors['border-color'],
+	appBorderRadius: 4,
+
+	// Text colors
+	textColor: theme.semanticTokens.colors['body-text'],
+	textInverseColor: theme.semanticTokens.colors['inverse-text'],
+	textMutedColor: theme.semanticTokens.colors['subtle-text'],
+
+	// Toolbar default and active colors
+	barTextColor: theme.semanticTokens.colors['body-text'],
+	barSelectedColor: theme.semanticTokens.colors['subtle-bg'],
+	barHoverColor: theme.semanticTokens.colors['subtle-bg'],
+	barBg: theme.semanticTokens.colors['body-bg'],
+
+	// Form colors
+	inputBg: theme.semanticTokens.colors['body-bg'],
+	inputBorder: theme.semanticTokens.colors['border-color'],
+	inputTextColor: theme.semanticTokens.colors['body-text'],
+	inputBorderRadius: 4,
+
+	// Grid 
+	gridCellSize: 6,
+});
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/styledComponents.ts b/datahub-web-react/.storybook/styledComponents.ts
new file mode 100644
index 0000000000000..5951c810d8998
--- /dev/null
+++ b/datahub-web-react/.storybook/styledComponents.ts
@@ -0,0 +1,36 @@
+import { createGlobalStyle } from 'styled-components';
+
+import '../src/fonts/Mulish-Regular.ttf';
+import '../src/fonts/Mulish-Medium.ttf';
+import '../src/fonts/Mulish-SemiBold.ttf';
+import '../src/fonts/Mulish-Bold.ttf';
+
+export const GlobalStyle = createGlobalStyle`
+    @font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 400;
+		src: url('../src/fonts/Mulish-Regular.ttf) format('truetype');
+    }
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 500;
+		src: url('../src/fonts/Mulish-Medium.ttf) format('truetype');
+	}
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 600;
+		src: url('../src/fonts/Mulish-SemiBold.ttf) format('truetype');
+	}
+	@font-face {
+		font-family: 'Mulish';
+		font-style: normal;
+		font-weight: 700;
+		src: url('../src/fonts/Mulish-Bold.ttf) format('truetype');
+	}
+	body {
+		font-family: 'Mulish', sans-serif;
+	}
+`;
\ No newline at end of file
diff --git a/datahub-web-react/.storybook/webpack.config.js b/datahub-web-react/.storybook/webpack.config.js
new file mode 100644
index 0000000000000..22e4ec1de6305
--- /dev/null
+++ b/datahub-web-react/.storybook/webpack.config.js
@@ -0,0 +1,13 @@
+const path = require('path');
+
+module.exports = {
+	module: {
+		loaders: [
+			{
+				test: /\.(png|woff|woff2|eot|ttf|svg)$/,
+				loaders: ['file-loader'],
+				include: path.resolve(__dirname, '../'),
+			},
+		],
+	},
+};
\ No newline at end of file
diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle
index b9fffce173c5c..bf1aa401e3f56 100644
--- a/datahub-web-react/build.gradle
+++ b/datahub-web-react/build.gradle
@@ -16,7 +16,7 @@ node {
   }
 
   // Version of node to use.
-  version = '21.2.0'
+  version = '22.12.0'
 
   // Version of Yarn to use.
   yarnVersion = '1.22.22'
@@ -93,7 +93,6 @@ task yarnLintFix(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
 }
 
 task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
-  environment = [NODE_OPTIONS: "--max-old-space-size=3072 --openssl-legacy-provider"]
   args = ['run', 'build']
 
   outputs.cacheIf { true }
diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
index dcaef6004d702..2d1d667a89f14 100644
--- a/datahub-web-react/package.json
+++ b/datahub-web-react/package.json
@@ -9,8 +9,12 @@
         "@ant-design/colors": "^5.0.0",
         "@ant-design/icons": "^4.3.0",
         "@apollo/client": "^3.3.19",
+        "@fontsource/mulish": "^5.0.16",
+        "@geometricpanda/storybook-addon-badges": "^2.0.2",
         "@graphql-codegen/fragment-matcher": "^5.0.0",
         "@monaco-editor/react": "^4.3.1",
+        "@mui/icons-material": "^5.15.21",
+        "@mui/material": "^5.15.21",
         "@react-hook/window-size": "^3.0.7",
         "@react-spring/web": "^9.7.3",
         "@remirror/pm": "^2.0.3",
@@ -30,6 +34,7 @@
         "@uiw/react-md-editor": "^3.3.4",
         "@visx/axis": "^3.1.0",
         "@visx/curve": "^3.0.0",
+        "@visx/gradient": "^3.3.0",
         "@visx/group": "^3.0.0",
         "@visx/hierarchy": "^3.0.0",
         "@visx/legend": "^3.2.0",
@@ -85,7 +90,7 @@
         "analyze": "source-map-explorer 'dist/assets/*.js'",
         "start": "yarn run generate && vite",
         "ec2-dev": "yarn run generate && CI=true;export CI;vite",
-        "build": "yarn run generate && NODE_OPTIONS='--max-old-space-size=3072 --openssl-legacy-provider' CI=false vite build",
+        "build": "yarn run generate && NODE_OPTIONS='--max-old-space-size=4096 --openssl-legacy-provider' CI=false vite build",
         "test": "vitest",
         "generate": "graphql-codegen --config codegen.yml",
         "lint": "eslint . --ext .ts,.tsx --quiet && yarn format-check && yarn type-check",
@@ -93,7 +98,9 @@
         "format-check": "prettier --check src",
         "format": "prettier --write src",
         "type-check": "tsc --noEmit",
-        "type-watch": "tsc -w --noEmit"
+        "type-watch": "tsc -w --noEmit",
+        "storybook": "storybook dev -p 6006",
+        "build-storybook": "storybook build"
     },
     "browserslist": {
         "production": [
@@ -112,6 +119,16 @@
         "@graphql-codegen/near-operation-file-preset": "^1.17.13",
         "@graphql-codegen/typescript-operations": "1.17.13",
         "@graphql-codegen/typescript-react-apollo": "2.2.1",
+        "@storybook/addon-essentials": "^8.1.11",
+        "@storybook/addon-interactions": "^8.1.11",
+        "@storybook/addon-links": "^8.1.11",
+        "@storybook/addon-onboarding": "^8.1.11",
+        "@storybook/blocks": "^8.1.11",
+        "@storybook/builder-vite": "^8.1.11",
+        "@storybook/manager-api": "^8.1.11",
+        "@storybook/react-vite": "^8.1.11",
+        "@storybook/test": "^8.1.11",
+        "@storybook/theming": "^8.1.11",
         "@types/graphql": "^14.5.0",
         "@types/query-string": "^6.3.0",
         "@types/styled-components": "^5.1.7",
@@ -132,6 +149,7 @@
         "less": "^4.2.0",
         "prettier": "^2.8.8",
         "source-map-explorer": "^2.5.2",
+        "storybook": "^8.1.11",
         "vite": "^4.5.5",
         "vite-plugin-babel-macros": "^1.0.6",
         "vite-plugin-static-copy": "^0.17.0",
diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx
index 2fdd7c8ed6800..81f137417f1f8 100644
--- a/datahub-web-react/src/App.tsx
+++ b/datahub-web-react/src/App.tsx
@@ -79,7 +79,7 @@ export const InnerApp: React.VFC = () => {
         <HelmetProvider>
             <CustomThemeProvider>
                 <Helmet>
-                    <title>{useCustomTheme().theme?.content.title}</title>
+                    <title>{useCustomTheme().theme?.content?.title}</title>
                 </Helmet>
                 <Router>
                     <Routes />
diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx
index aed672a34e7ca..73a789030ce6f 100644
--- a/datahub-web-react/src/Mocks.tsx
+++ b/datahub-web-react/src/Mocks.tsx
@@ -566,6 +566,12 @@ export const dataset3 = {
                     username: 'datahub',
                     type: EntityType.CorpUser,
                 },
+                actor: {
+                    __typename: 'CorpUser',
+                    urn: 'urn:li:corpuser:datahub',
+                    username: 'datahub',
+                    type: EntityType.CorpUser,
+                },
                 description: 'This only points to Google',
                 label: 'This only points to Google',
                 created: {
@@ -2198,7 +2204,7 @@ export const mocks = [
                     count: 10,
                     filters: [],
                     orFilters: [],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -2238,6 +2244,7 @@ export const mocks = [
                             field: 'origin',
                             displayName: 'origin',
                             aggregations: [{ value: 'PROD', count: 3, entity: null }],
+                            entity: null,
                         },
                         {
                             field: '_entityType',
@@ -2246,6 +2253,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -2255,6 +2263,7 @@ export const mocks = [
                                 { value: 'MySQL', count: 1, entity: null },
                                 { value: 'Kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                     suggestions: [],
@@ -2284,7 +2293,7 @@ export const mocks = [
                             ],
                         },
                     ],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -2319,6 +2328,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: '_entityType',
@@ -2327,6 +2337,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             __typename: 'FacetMetadata',
@@ -2337,6 +2348,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                     suggestions: [],
@@ -2387,6 +2399,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: '_entityType',
@@ -2395,6 +2408,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -2404,6 +2418,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -2458,7 +2473,7 @@ export const mocks = [
                             ],
                         },
                     ],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -2495,6 +2510,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             __typename: 'FacetMetadata',
@@ -2504,6 +2520,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             __typename: 'FacetMetadata',
@@ -2514,6 +2531,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' },
                                 { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -2663,6 +2681,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: '_entityType',
@@ -2671,6 +2690,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -2680,6 +2700,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -2737,6 +2758,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: '_entityType',
@@ -2745,6 +2767,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -2754,6 +2777,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -2803,6 +2827,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -2816,6 +2841,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -2947,6 +2973,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -2960,6 +2987,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -3007,7 +3035,7 @@ export const mocks = [
                             ],
                         },
                     ],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -3044,6 +3072,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         // {
                         //     displayName: 'Domain',
@@ -3065,6 +3094,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             __typename: 'FacetMetadata',
@@ -3090,6 +3120,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -3175,7 +3206,7 @@ export const mocks = [
                             ],
                         },
                     ],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -3209,6 +3240,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -3222,6 +3254,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -3250,7 +3283,7 @@ export const mocks = [
                             ],
                         },
                     ],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -3284,6 +3317,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: '_entityType',
@@ -3292,6 +3326,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -3301,6 +3336,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -3329,7 +3365,7 @@ export const mocks = [
                             ],
                         },
                     ],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -3371,6 +3407,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: '_entityType',
@@ -3379,6 +3416,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -3388,6 +3426,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -3422,7 +3461,7 @@ export const mocks = [
                             ],
                         },
                     ],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -3459,6 +3498,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             __typename: 'FacetMetadata',
@@ -3468,6 +3508,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             __typename: 'FacetMetadata',
@@ -3478,6 +3519,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' },
                                 { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -3512,7 +3554,7 @@ export const mocks = [
                             ],
                         },
                     ],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -3549,6 +3591,7 @@ export const mocks = [
                                     __typename: 'AggregationMetadata',
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             __typename: 'FacetMetadata',
@@ -3558,6 +3601,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             __typename: 'FacetMetadata',
@@ -3568,6 +3612,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' },
                                 { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -3629,6 +3674,8 @@ export const mocks = [
                         manageGlobalAnnouncements: true,
                         createBusinessAttributes: true,
                         manageBusinessAttributes: true,
+                        manageStructuredProperties: true,
+                        viewStructuredPropertiesPage: true,
                     },
                 },
             },
@@ -3716,7 +3763,7 @@ export const mocks = [
                     count: 10,
                     filters: [],
                     orFilters: [],
-                    searchFlags: { getSuggestions: true },
+                    searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
                 },
             },
         },
@@ -3815,6 +3862,7 @@ export const mocks = [
                                     entity: null,
                                 },
                             ],
+                            entity: null,
                         },
                         {
                             field: '_entityType',
@@ -3823,6 +3871,7 @@ export const mocks = [
                                 { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' },
                                 { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' },
                             ],
+                            entity: null,
                         },
                         {
                             field: 'platform',
@@ -3832,6 +3881,7 @@ export const mocks = [
                                 { value: 'mysql', count: 1, entity: null },
                                 { value: 'kafka', count: 1, entity: null },
                             ],
+                            entity: null,
                         },
                     ],
                 },
@@ -3906,4 +3956,6 @@ export const platformPrivileges: PlatformPrivileges = {
     manageGlobalAnnouncements: true,
     createBusinessAttributes: true,
     manageBusinessAttributes: true,
+    manageStructuredProperties: true,
+    viewStructuredPropertiesPage: true,
 };
diff --git a/datahub-web-react/src/alchemy-components/.docs/Contributing.mdx b/datahub-web-react/src/alchemy-components/.docs/Contributing.mdx
new file mode 100644
index 0000000000000..75a31d011903f
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/Contributing.mdx
@@ -0,0 +1,43 @@
+import { Meta } from '@storybook/blocks';
+
+<Meta title="Contributing" />
+
+<div className="custom-docs">
+	## Contributing
+
+    Building and maintinging a design system is a collaborative effort. We welcome contributions from all team members, regardless of their role or experience level. This document outlines the process for contributing to the Acryl Component Library.
+
+    ### Development
+
+    To run Storybook locally, use the following command:
+
+    ```
+    yarn storybook
+    ```
+
+    Storybook will start a local development server and open a new browser window with the Storybook interface on port `6006`. When developing new components or updating existing ones, you can use Storybook to preview your changes in real-time. This will ensure that the component looks and behaves as expected before merging your changes.
+
+    ### Crafting New Components
+
+    When creating new components, make sure to follow the established design patterns and coding standards. This will help maintain consistency across all Acryl products and make it easier for other team members to understand and use your components.
+
+    Design new components with <strong>reusability in mind</strong>. Components should be flexible, extensible, and easy to customize. Avoid hardcoding values and use props to pass data and styles to your components. This will make it easier to reuse the component in different contexts and scenarios.
+
+    Our design team works exclusively in Figma, so if questions arise about the design or implementation of a component, please refer to the Figma files for more information. If you have any questions or need clarification, feel free to reach out to the design team for assistance.
+
+    ### Pull Requests
+
+    When submitting a pull request, please follow these guidelines:
+
+    1. Create a new branch for your changes.
+    2. Make sure your code is well-documented and follows the established coding standards.
+    3. Write clear and concise commit messages.
+    4. Include a detailed description of the changes in your pull request.
+
+    If applicable, include screenshots or GIFs to demonstrate the changes visually. This will help reviewers understand the context of your changes and provide more accurate feedback. If a Figma file exists, include a link to the file in the pull request description.
+
+    ### Review Process
+
+    All pull requests will be reviewed by the UI and design team to ensure that the changes align with the design system guidelines and best practices. The team will provide feedback and suggestions for improvement, and you may be asked to make additional changes before your pull request is merged.
+
+</div>
diff --git a/datahub-web-react/src/alchemy-components/.docs/DesignTokens.mdx b/datahub-web-react/src/alchemy-components/.docs/DesignTokens.mdx
new file mode 100644
index 0000000000000..0ebdebbf9db4c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/DesignTokens.mdx
@@ -0,0 +1,63 @@
+import { Meta, Source } from '@storybook/blocks';
+
+import theme from '@components/theme';
+
+import { ColorCard, CopyButton } from './mdx-components';
+
+<Meta title="Design Tokens" />
+
+<div className="custom-docs">
+	## Design Tokens
+
+    To streamline the design process and ensure consistency across all Acryl products, we use a set of design tokens that define the visual properties of our design system. These tokens include colors, typography, spacing, and other visual elements that can be used to create a cohesive user experience.
+
+    ### Colors
+
+    ```tsx
+    import theme from '@components/theme';
+
+    // Accessing a color via object path
+    <div style={{ color: theme.semanticTokens.colors.primary }}>Hello, World!</div>
+
+    // Using CSS variables
+    <div style={{ color: 'var(--alch-color-primary)' }}>Hello, World!</div>
+    ```
+
+    <table style={{ width: '100%' }}>
+    	<thead style={{ textAlign: 'left' }}>
+    		<tr>
+    			<th>Token Value</th>
+    			<th>Selector</th>
+    			<th>CSS Variable <small>(coming soon)</small></th>
+    		</tr>
+    	</thead>
+    	<tbody>
+    		{Object.keys(theme.semanticTokens.colors).map((color) => {
+    			const objectKey = `colors['${color}']`;
+    			const hexValue = theme.semanticTokens.colors[color];
+    			const cssVar = `--alch-color-${color}`;
+
+    			return (
+    				<tr key={color}>
+    					<td>
+    						<ColorCard color={hexValue} size="sm">
+    							<span className="colorChip" />
+    							<div>
+    								<span className="colorValue">{color}</span>
+    								<span className="hex">{hexValue}</span>
+    							</div>
+    						</ColorCard>
+    					</td>
+    					<td>
+    						<span style={{ display: 'flex', alignItems: 'center', fontSize: 'inherit' }}>
+    							{objectKey} <CopyButton text={objectKey} />
+    						</span>
+    					</td>
+    					<td>{cssVar}</td>
+    				</tr>
+    			);
+    		})}
+    	</tbody>
+    </table>
+
+</div>
diff --git a/datahub-web-react/src/alchemy-components/.docs/Icons.mdx b/datahub-web-react/src/alchemy-components/.docs/Icons.mdx
new file mode 100644
index 0000000000000..e3f6ab6846119
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/Icons.mdx
@@ -0,0 +1,34 @@
+import { Meta, Source } from '@storybook/blocks';
+
+import { AVAILABLE_ICONS } from '@components';
+import { IconGalleryWithSearch } from './mdx-components';
+
+<Meta title="Icons" />
+
+<div className="custom-docs">
+	## Icons
+
+    Under the hood, we're utilizing the Material Design Icon Library. However, we've crafted out own resuable component to make it easier to use these icons in our application.
+
+    <a href="/?path=/docs/media-icon--docs">
+    	View the component documentation to learn more
+    </a>
+
+    In addition to using Materials Design Icons, we've also added a few custom icons to the library. You can access them through the same `<Icon />` component and are represented in the list of available options below.
+
+    ```tsx
+    import { Icon } from '@components';
+
+    <Icon icon="AccountCircle" />
+    ```
+
+    <br />
+
+    ### Gallery
+
+    There are {AVAILABLE_ICONS.length} icons available. <br />
+    Name values populate the `icon` prop on the `<Icon />` component.
+
+    <IconGalleryWithSearch icons={AVAILABLE_ICONS} />
+
+</div>
diff --git a/datahub-web-react/src/alchemy-components/.docs/Intro.mdx b/datahub-web-react/src/alchemy-components/.docs/Intro.mdx
new file mode 100644
index 0000000000000..f81d08059c7b4
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/Intro.mdx
@@ -0,0 +1,14 @@
+import { Meta, Description } from '@storybook/blocks';
+import ReadMe from '../README.mdx';
+
+<Meta title="Introduction" />
+
+<div className="custom-docs">
+	<div className="acrylBg">
+		<img src="https://www.acryldata.io/images/logo-pattern.svg" alt="Acryl Logo" />
+	</div>
+
+    {/* To simply, we're rendering the root readme here */}
+    <ReadMe />
+
+</div>
diff --git a/datahub-web-react/src/alchemy-components/.docs/StyleGuide.mdx b/datahub-web-react/src/alchemy-components/.docs/StyleGuide.mdx
new file mode 100644
index 0000000000000..43199cbbca62d
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/StyleGuide.mdx
@@ -0,0 +1,209 @@
+import { Meta, Source } from '@storybook/blocks';
+
+import { Heading } from '@components';
+import { colors } from '@components/theme';
+
+import { Grid, FlexGrid, ColorCard, CopyButton, Seperator } from './mdx-components';
+
+import borderSource from '@components/theme/foundations/borders?raw';
+import colorsSource from '@components/theme/foundations/colors?raw';
+import typographySource from '@components/theme/foundations/typography?raw';
+import radiusSource from '@components/theme/foundations/radius?raw';
+import shadowsSource from '@components/theme/foundations/shadows?raw';
+import sizesSource from '@components/theme/foundations/sizes?raw';
+import spacingSource from '@components/theme/foundations/spacing?raw';
+import transitionSource from '@components/theme/foundations/transition?raw';
+import zIndexSource from '@components/theme/foundations/zIndex?raw';
+
+<Meta title="Style Guide" />
+
+<div className="custom-docs">
+	## Style Guide
+
+    The purpose of this Style Guide is to establish a unified and cohesive design language that ensures a consistent user experience across all Acryl products. By adhering to these guidelines, we can maintain a high standard of design quality and improve the usability of our applications.
+
+    ### Theme
+
+    You can import the theme object into any component or file in your application and use it to style your components. The theme object is a single source of truth for your application's design system.
+
+    ```tsx
+    import { typography, colors, spacing } from '@components/theme';
+    ```
+
+    ### Colors
+
+    Colors are managed via the `colors.ts` file in the `theme/foundations` directory. The colors are defined as a nested object with the following structure:
+
+    <Source code={colorsSource} />
+
+    By default, all `500` values are considered the "default" value of that color range. For example, `gray.500` is the default gray color. The other values are used for shading and highlighting. Color values are defined in hex format and their values range between 25 and 1000. With 25 being the lighest and 1000 being the darkest.
+
+    #### Black & White
+    <FlexGrid>
+    	<ColorCard color={colors['black']}>
+    		<span className="colorChip" />
+    		<div>
+    			<span className="colorValue">Black</span>
+    			<span className="hex">{colors['black']}</span>
+    		</div>
+    	</ColorCard>
+    	<ColorCard color={colors['white']}>
+    		<span className="colorChip" />
+    		<div>
+    			<span className="colorValue">White</span>
+    			<span className="hex">{colors['white']}</span>
+    		</div>
+    	</ColorCard>
+    </FlexGrid>
+
+    <Seperator />
+
+    #### Gray
+    <Grid>
+    	{Object.keys(colors.gray).map((color) => (
+    		<ColorCard key={color} color={colors['gray'][color]}>
+    			<span className="colorChip" />
+    			<div>
+    				<span className="colorValue">
+    					Gray {color} <CopyButton text={`gray.${color}`} />
+    				</span>
+    				<span className="hex">{colors['gray'][color]}</span>
+    			</div>
+    		</ColorCard>
+    	))}
+    </Grid>
+
+    <Seperator />
+
+    #### Violet (Primary)
+    <Grid>
+    	{Object.keys(colors.violet).map((color) => (
+    		<ColorCard key={color} color={colors['violet'][color]}>
+    			<span className="colorChip" />
+    			<div>
+    				<span className="colorValue">
+    					Violet {color} <CopyButton text={`violet.${color}`} />
+    				</span>
+    				<span className="hex">{colors['violet'][color]}</span>
+    			</div>
+    		</ColorCard>
+    	))}
+    </Grid>
+
+    <Seperator />
+
+    #### Blue
+    <Grid>
+    	{Object.keys(colors.blue).map((color) => (
+    		<ColorCard key={color} color={colors['blue'][color]}>
+    			<span className="colorChip" />
+    			<div>
+    				<span className="colorValue">
+    					Blue {color} <CopyButton text={`blue.${color}`} />
+    				</span>
+    				<span className="hex">{colors['blue'][color]}</span>
+    			</div>
+    		</ColorCard>
+    	))}
+    </Grid>
+
+    <Seperator />
+
+    #### Green
+    <Grid>
+    	{Object.keys(colors.green).map((color) => (
+    		<ColorCard key={color} color={colors['green'][color]}>
+    			<span className="colorChip" />
+    			<div>
+    				<span className="colorValue">
+    					Green {color} <CopyButton text={`green.${color}`} />
+    				</span>
+    				<span className="hex">{colors['green'][color]}</span>
+    			</div>
+    		</ColorCard>
+    	))}
+    </Grid>
+
+    <Seperator />
+
+    #### Yellow
+    <Grid>
+    	{Object.keys(colors.yellow).map((color) => (
+    		<ColorCard key={color} color={colors['yellow'][color]}>
+    			<span className="colorChip" />
+    			<div>
+    				<span className="colorValue">
+    					Yellow {color} <CopyButton text={`yellow.${color}`} />
+    				</span>
+    				<span className="hex">{colors['yellow'][color]}</span>
+    			</div>
+    		</ColorCard>
+    	))}
+    </Grid>
+
+    <Seperator />
+
+    #### Red
+    <Grid>
+    	{Object.keys(colors.red).map((color) => (
+    		<ColorCard key={color} color={colors['red'][color]}>
+    			<span className="colorChip" />
+    			<div>
+    				<span className="colorValue">
+    					Red {color} <CopyButton text={`red.${color}`} />
+    				</span>
+    				<span className="hex">{colors['red'][color]}</span>
+    			</div>
+    		</ColorCard>
+    	))}
+    </Grid>
+
+    ### Typography
+
+    Font styles are managed via the `typography.ts` file in the `theme/foundations` directory. The primary font family in use is `Mulish`. The font styles are defined as a nested object with the following structure:
+
+    <Source code={typographySource} />
+
+    ### Borders
+
+    A set of border values defined by the border key.
+
+    <Source code={borderSource} />
+
+    ### Border Radius
+
+    A set smooth corner radius values defined by the radii key.
+
+    <Source code={radiusSource} />
+
+    ### Shadows
+
+    A set of shadow values defined by the shadows key.
+
+    <Source code={shadowsSource} />
+
+    ## Sizes
+
+    A set of size values defined by the sizes key.
+
+    <Source code={sizesSource} />
+
+    ### Spacing
+
+    A set of spacing values defined by the spacing key.
+
+    <Source code={spacingSource} />
+
+    ### Transitions
+
+    A set of transition values defined by the transition key.
+
+    <Source code={transitionSource} />
+
+    ### Z-Index
+
+    A set of z-index values defined by the zindex key.
+
+    <Source code={zIndexSource} />
+
+</div>
diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/CodeBlock.tsx b/datahub-web-react/src/alchemy-components/.docs/mdx-components/CodeBlock.tsx
new file mode 100644
index 0000000000000..43b9ebfae6414
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/CodeBlock.tsx
@@ -0,0 +1,24 @@
+import React from 'react';
+
+import { Source, DocsContext } from '@storybook/blocks';
+
+export const CodeBlock = () => {
+    const context = React.useContext(DocsContext);
+
+    const { primaryStory } = context as any;
+    const component = context ? primaryStory.component.__docgenInfo.displayName : '';
+
+    if (!context || !primaryStory) return null;
+
+    return (
+        <div>
+            <Source
+                code={`
+					import { ${component} } from '@components';
+				`}
+                format
+                dark
+            />
+        </div>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/CopyButton.tsx b/datahub-web-react/src/alchemy-components/.docs/mdx-components/CopyButton.tsx
new file mode 100644
index 0000000000000..c81aa6ed44289
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/CopyButton.tsx
@@ -0,0 +1,16 @@
+import React from 'react';
+
+import { Button, Icon } from '@components';
+import { copyToClipboard } from './utils';
+
+interface Props {
+    text: string;
+}
+
+export const CopyButton = ({ text }: Props) => (
+    <div style={{ display: 'inline-block' }}>
+        <Button variant="text" color="gray" size="sm" onClick={() => copyToClipboard(text)}>
+            <Icon icon="ContentCopy" size="xs" />
+        </Button>
+    </div>
+);
diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/GridList.tsx b/datahub-web-react/src/alchemy-components/.docs/mdx-components/GridList.tsx
new file mode 100644
index 0000000000000..5cb4bd27e521a
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/GridList.tsx
@@ -0,0 +1,32 @@
+/* 
+ Docs Only Component that helps to display a list of components in a grid layout.
+*/
+
+import React, { ReactNode } from 'react';
+
+const styles = {
+    display: 'flex',
+    alignItems: 'center',
+    justifyContent: 'center',
+    gap: '8px',
+};
+
+interface Props {
+    isVertical?: boolean;
+    width?: number | string;
+    children: ReactNode;
+}
+
+export const GridList = ({ isVertical = false, width = '100%', children }: Props) => {
+    return (
+        <div
+            style={{
+                ...styles,
+                width,
+                flexDirection: isVertical ? 'column' : 'row',
+            }}
+        >
+            {children}
+        </div>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/IconGalleryWithSearch.tsx b/datahub-web-react/src/alchemy-components/.docs/mdx-components/IconGalleryWithSearch.tsx
new file mode 100644
index 0000000000000..d8751509bd6a7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/IconGalleryWithSearch.tsx
@@ -0,0 +1,291 @@
+import React, { useState } from 'react';
+
+import { Icon, Button, ButtonProps } from '@components';
+import { IconGrid, IconGridItem, IconDisplayBlock } from './components';
+
+interface Props {
+    icons: string[];
+}
+
+export const IconGalleryWithSearch = ({ icons }: Props) => {
+    const [iconSet, setIconSet] = useState(icons);
+    const [search, setSearch] = useState('');
+    const [variant, setVariant] = useState('outline');
+
+    const filteredIcons = iconSet.filter((icon) => icon.toLowerCase().includes(search.toLowerCase()));
+
+    const arrows = [
+        'ArrowBack',
+        'ArrowCircleDown',
+        'ArrowCircleLeft',
+        'ArrowCircleRight',
+        'ArrowCircleUp',
+        'ArrowDownward',
+        'ArrowForward',
+        'ArrowOutward',
+        'ArrowUpward',
+        'CloseFullscreen',
+        'Cached',
+        'Code',
+        'CodeOff',
+        'CompareArrows',
+        'Compress',
+        'ChevronLeft',
+        'ChevronRight',
+        'DoubleArrow',
+        'FastForward',
+        'FastRewind',
+        'FileDownload',
+        'FileUpload',
+        'ForkLeft',
+        'ForkRight',
+        'GetApp',
+        'LastPage',
+        'Launch',
+        'Login',
+        'Logout',
+        'LowPriority',
+        'ManageHistory',
+        'Merge',
+        'MergeType',
+        'MoveUp',
+        'MultipleStop',
+        'OpenInFull',
+        'Outbound',
+        'Outbox',
+        'Output',
+        'PlayArrow',
+        'PlayCircle',
+        'Publish',
+        'ReadMore',
+        'ExitToApp',
+        'Redo',
+        'Refresh',
+        'Replay',
+        'ReplyAll',
+        'Reply',
+        'Restore',
+        'SaveAlt',
+        'Shortcut',
+        'SkipNext',
+        'SkipPrevious',
+        'Start',
+        'Straight',
+        'SubdirectoryArrowLeft',
+        'SubdirectoryArrowRight',
+        'SwapHoriz',
+        'SwapVert',
+        'SwitchLeft',
+        'SwitchRight',
+        'SyncAlt',
+        'SyncDisabled',
+        'SyncLock',
+        'Sync',
+        'Shuffle',
+        'SyncProblem',
+        'TrendingDown',
+        'TrendingFlat',
+        'TrendingUp',
+        'TurnLeft',
+        'TurnRight',
+        'TurnSlightLeft',
+        'TurnSlightRight',
+        'Undo',
+        'UnfoldLessDouble',
+        'UnfoldLess',
+        'UnfoldMoreDouble',
+        'UnfoldMore',
+        'UpdateDisabled',
+        'Update',
+        'Upgrade',
+        'Upload',
+        'ZoomInMap',
+        'ZoomOutMap',
+    ];
+
+    const dataViz = [
+        'AccountTree',
+        'Analytics',
+        'ArtTrack',
+        'Article',
+        'BackupTable',
+        'BarChart',
+        'BubbleChart',
+        'Calculate',
+        'Equalizer',
+        'List',
+        'FormatListBulleted',
+        'FormatListNumbered',
+        'Grading',
+        'InsertChart',
+        'Hub',
+        'Insights',
+        'Lan',
+        'Leaderboard',
+        'LegendToggle',
+        'Map',
+        'MultilineChart',
+        'Nat',
+        'PivotTableChart',
+        'Poll',
+        'Polyline',
+        'QueryStats',
+        'Radar',
+        'Route',
+        'Rule',
+        'Schema',
+        'Sort',
+        'SortByAlpha',
+        'ShowChart',
+        'Source',
+        'SsidChart',
+        'StackedBarChart',
+        'StackedLineChart',
+        'Storage',
+        'TableChart',
+        'TableRows',
+        'TableView',
+        'Timeline',
+        'ViewAgenda',
+        'ViewArray',
+        'ViewCarousel',
+        'ViewColumn',
+        'ViewComfy',
+        'ViewCompact',
+        'ViewCozy',
+        'ViewDay',
+        'ViewHeadline',
+        'ViewKanban',
+        'ViewList',
+        'ViewModule',
+        'ViewQuilt',
+        'ViewSidebar',
+        'ViewStream',
+        'ViewTimeline',
+        'ViewWeek',
+        'Visibility',
+        'VisibilityOff',
+        'Webhook',
+        'Window',
+    ];
+
+    const social = [
+        'AccountCircle',
+        'Badge',
+        'Campaign',
+        'Celebration',
+        'Chat',
+        'ChatBubble',
+        'CommentBank',
+        'Comment',
+        'CommentsDisabled',
+        'Message',
+        'ContactPage',
+        'Contacts',
+        'GroupAdd',
+        'Group',
+        'GroupRemove',
+        'Groups',
+        'Handshake',
+        'ManageAccounts',
+        'MoodBad',
+        'SentimentDissatisfied',
+        'SentimentNeutral',
+        'SentimentSatisfied',
+        'Mood',
+        'NoAccounts',
+        'People',
+        'PersonAddAlt1',
+        'PersonOff',
+        'Person',
+        'PersonRemoveAlt1',
+        'PersonSearch',
+        'SwitchAccount',
+        'StarBorder',
+        'StarHalf',
+        'Star',
+        'ThumbDown',
+        'ThumbUp',
+        'ThumbsUpDown',
+        'Verified',
+        'VerifiedUser',
+    ];
+
+    const notifs = [
+        'Mail',
+        'Drafts',
+        'MarkAsUnread',
+        'Inbox',
+        'Outbox',
+        'MoveToInbox',
+        'Unsubscribe',
+        'Upcoming',
+        'NotificationAdd',
+        'NotificationImportant',
+        'NotificationsActive',
+        'NotificationsOff',
+        'Notifications',
+        'NotificationsPaused',
+    ];
+
+    const handleChangeSet = (set) => {
+        setIconSet(set);
+        setSearch('');
+    };
+
+    const handleResetSet = () => {
+        setIconSet(icons);
+        setSearch('');
+    };
+
+    const smButtonProps: ButtonProps = {
+        size: 'sm',
+        color: 'gray',
+    };
+
+    return (
+        <>
+            <input
+                type="search"
+                value={search}
+                onChange={(e) => setSearch(e.target.value)}
+                placeholder="Search for an icon…"
+                style={{ width: '100%', padding: '0.5rem', marginBottom: '0.5rem' }}
+            />
+            <div style={{ display: 'flex', justifyContent: 'space-between', gap: '8px' }}>
+                <div style={{ display: 'flex', gap: '8px' }}>
+                    <Button onClick={handleResetSet} {...smButtonProps}>
+                        All
+                    </Button>
+                    <Button onClick={() => handleChangeSet(arrows)} {...smButtonProps}>
+                        Arrows
+                    </Button>
+                    <Button onClick={() => handleChangeSet(dataViz)} {...smButtonProps}>
+                        Data Viz
+                    </Button>
+                    <Button onClick={() => handleChangeSet(social)} {...smButtonProps}>
+                        Social
+                    </Button>
+                    <Button onClick={() => handleChangeSet(notifs)} {...smButtonProps}>
+                        Notifications
+                    </Button>
+                </div>
+                <div style={{ display: 'flex', gap: '8px' }}>
+                    <Button onClick={() => setVariant(variant === 'outline' ? 'filled' : 'outline')} {...smButtonProps}>
+                        Variant: {variant === 'filled' ? 'Filled' : 'Outline'}
+                    </Button>
+                </div>
+            </div>
+            <IconGrid>
+                {filteredIcons.map((icon) => (
+                    <IconGridItem>
+                        <IconDisplayBlock key={icon} title={icon}>
+                            <Icon icon={icon} variant={variant as any} size="2xl" />
+                        </IconDisplayBlock>
+                        <span>{icon}</span>
+                    </IconGridItem>
+                ))}
+            </IconGrid>
+        </>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/components.ts b/datahub-web-react/src/alchemy-components/.docs/mdx-components/components.ts
new file mode 100644
index 0000000000000..28d428493b17b
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/components.ts
@@ -0,0 +1,110 @@
+/* 
+	Docs Only Components that helps to display information in info guides.
+*/
+
+import styled from 'styled-components';
+
+import theme from '@components/theme';
+
+export const Grid = styled.div`
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+    gap: 16px;
+`;
+
+export const FlexGrid = styled.div`
+    display: flex;
+    gap: 16px;
+`;
+
+export const VerticalFlexGrid = styled.div`
+    display: flex;
+    flex-direction: column;
+    gap: 16px;
+`;
+
+export const Seperator = styled.div`
+    height: 16px;
+`;
+
+export const ColorCard = styled.div<{ color: string; size?: string }>`
+    display: flex;
+    gap: 16px;
+    align-items: center;
+
+    ${({ size }) =>
+        size === 'sm' &&
+        `
+		gap: 8px;
+	`}
+
+    & span {
+        display: block;
+        line-height: 1.3;
+    }
+
+    & .colorChip {
+        background: ${({ color }) => color};
+        width: 3rem;
+        height: 3rem;
+
+        ${({ size }) =>
+            size === 'sm' &&
+            `
+			width: 2rem;
+			height: 2rem;
+			border-radius: 4px;
+		`}
+
+        border-radius: 8px;
+        box-shadow: rgba(0, 0, 0, 0.06) 0px 2px 4px 0px inset;
+    }
+
+    & .colorValue {
+        display: flex;
+        align-items: center;
+        gap: 0;
+        font-weight: bold;
+        font-size: 14px;
+    }
+
+    & .hex {
+        font-size: 11px;
+        opacity: 0.5;
+        text-transform: uppercase;
+    }
+`;
+
+export const IconGrid = styled.div`
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(100px, 1fr));
+    gap: 16px;
+    margin-top: 20px;
+`;
+
+export const IconGridItem = styled.div`
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: space-between;
+
+    border: 1px solid ${theme.semanticTokens.colors['border-color']};
+    border-radius: 8px;
+    overflow: hidden;
+
+    & span {
+        width: 100%;
+        border-top: 1px solid ${theme.semanticTokens.colors['border-color']};
+        background-color: ${theme.semanticTokens.colors['subtle-bg']};
+        text-align: center;
+        padding: 4px 8px;
+        font-size: 10px;
+    }
+`;
+
+export const IconDisplayBlock = styled.div`
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 50px;
+`;
diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/index.ts b/datahub-web-react/src/alchemy-components/.docs/mdx-components/index.ts
new file mode 100644
index 0000000000000..d1c1848d1eb37
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/index.ts
@@ -0,0 +1,6 @@
+export * from './CodeBlock';
+export * from './CopyButton';
+export * from './GridList';
+export * from './IconGalleryWithSearch';
+export * from './components';
+export * from './utils';
diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/utils.ts b/datahub-web-react/src/alchemy-components/.docs/mdx-components/utils.ts
new file mode 100644
index 0000000000000..d4fa47dc9e967
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/utils.ts
@@ -0,0 +1,15 @@
+/*
+	Docs related utils 
+*/
+
+/**
+ * Copies the given text to the clipboard.
+ * @param {string} text - The text to be copied to the clipboard.
+ * @returns {Promise<void>} A promise that resolves when the text is copied.
+ */
+export const copyToClipboard = (text: string) => {
+    return navigator.clipboard
+        .writeText(text)
+        .then(() => console.log(`${text} copied to clipboard`))
+        .catch();
+};
diff --git a/datahub-web-react/src/alchemy-components/README.mdx b/datahub-web-react/src/alchemy-components/README.mdx
new file mode 100644
index 0000000000000..5373432c0ede0
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/README.mdx
@@ -0,0 +1,73 @@
+# Alchemy Component Library
+
+This is a comprehensive library of accessible and reusable React components that streamlines the development of Acryl's applications and websites. The library offers a diverse range of components that can be easily combined to build complex user interfaces while adhering to accessibility best practices.
+
+### Component Usage
+
+It's easy to use the components availble in the library. Simply import the component and use it anywhere you're rendering React components.
+
+```tsx
+import { Button } from '@components';
+
+function YourComponent() {
+    return <Button>Click me!</Button>;
+}
+```
+
+In addition to the components themselves, you can also import their types:
+
+```tsx
+import type { ButtonProps } from '@components';
+```
+
+### Theme Usage
+
+This component library comes with a complete theme utility that pre-defines all of our styling atoms and makes them accessible at `@components/theme`.
+
+```tsx
+import { colors } from '@components/theme';
+
+function YourComponent() {
+	return (
+		<div style={{ bgColor: colors.green.400 }}>
+			This div has a green background!
+		</div>
+	)
+}
+```
+
+You can access the theme types at `@components/theme/types` and the theme config at `@components/theme/config`.
+
+### Writing Docs
+
+Our docs are generated using [Storybook](https://storybook.js.org/) and deployed to [Cloudfare](https://www.cloudflare.com/).
+
+-   Storybook config is located at `.storybook`
+-   Static doc files are located at `alchemy-components/.docs`
+-   Component stories are located in each component directory: <br/>`alchemy-components/components/Component/Component.stories.tsx`
+
+Storybook serves as our playground for developing components. You can start it locally:
+
+```bash
+yarn storybook
+```
+
+This launches the docs app at `localhost:6006` and enables everything you need to quickly develop and document components.
+
+### Contributing
+
+Building a component library is a collaboriate effort! We're aiming to provide a first-class experience, so here's a list of the standards we'll be looking for:
+
+-   Consitent prop and variant naming conventions: <br />
+    -- `variant` is used to define style types, such as `outline` or `filled`. <br />
+    -- `color` is used to define the components color, such as `violet` or `blue`. <br />
+    -- `size` is used to define the components size, such as `xs` or `4xl`. <br />
+    -- Booleans are prefixed with `is`: `isLoading` or `isDisabled`.
+-   All style props have a correseponding theme type, ie. `FontSizeOptions`.
+-   All components have an export of default props.
+-   Styles are defined using `style objects` instead of `tagged template literals`.
+-   Stories are organized into the correct directory .
+
+### FAQs
+
+-   **How are components being styled?** <br />Our components are built using [Styled Components](https://styled-components.com/) that dynamically generate styles based on variant selection.
diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.stories.tsx b/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.stories.tsx
new file mode 100644
index 0000000000000..09d0d37f15421
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.stories.tsx
@@ -0,0 +1,133 @@
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import { GridList } from '@src/alchemy-components/.docs/mdx-components';
+import { Meta, StoryObj } from '@storybook/react';
+import React from 'react';
+import { Avatar, avatarDefaults } from './Avatar';
+
+const IMAGE_URL =
+    'https://is1-ssl.mzstatic.com/image/thumb/Purple211/v4/78/cb/e1/78cbe16d-28d9-057e-9f73-524c32eb5fe5/AppIcon-0-0-1x_U007emarketing-0-7-0-85-220.png/512x512bb.jpg';
+
+// Auto Docs
+const meta = {
+    title: 'Components / Avatar',
+    component: Avatar,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'This component allows users to render a user pill with picture and name',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        name: {
+            description: 'Name of the user.',
+            table: {
+                defaultValue: { summary: `${avatarDefaults.name}` },
+            },
+            control: 'text',
+        },
+        imageUrl: {
+            description: 'URL of the user image.',
+            control: 'text',
+        },
+        onClick: {
+            description: 'On click function for the Avatar.',
+        },
+        size: {
+            description: 'Size of the Avatar.',
+            table: {
+                defaultValue: { summary: `${avatarDefaults.size}` },
+            },
+            control: 'select',
+        },
+        showInPill: {
+            description: 'Whether Avatar is shown in pill format with name.',
+            table: {
+                defaultValue: { summary: `${avatarDefaults.showInPill}` },
+            },
+            control: 'boolean',
+        },
+
+        isOutlined: {
+            description: 'Whether Avatar is outlined.',
+            table: {
+                defaultValue: { summary: `${avatarDefaults.isOutlined}` },
+            },
+            control: 'boolean',
+        },
+    },
+
+    // Define defaults
+    args: {
+        name: 'John Doe',
+        size: 'default',
+        showInPill: false,
+        isOutlined: false,
+    },
+} satisfies Meta<typeof Avatar>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+// Basic story is what is displayed 1st in storybook & is used as the code sandbox
+// Pass props to this so that it can be customized via the UI props panel
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Avatar {...props} />,
+};
+
+export const sizes = () => (
+    <GridList>
+        <Avatar name="John Doe" size="lg" />
+        <Avatar name="John Doe" size="md" />
+        <Avatar name="John Doe" size="default" />
+        <Avatar name="John Doe" size="sm" />
+    </GridList>
+);
+
+export const withImage = () => (
+    <GridList>
+        <Avatar name="John Doe" imageUrl={IMAGE_URL} size="lg" />
+        <Avatar name="John Doe" imageUrl={IMAGE_URL} size="md" />
+        <Avatar name="John Doe" imageUrl={IMAGE_URL} size="default" />
+        <Avatar name="John Doe" imageUrl={IMAGE_URL} size="sm" />
+    </GridList>
+);
+
+export const pills = () => (
+    <GridList isVertical>
+        <GridList>
+            <Avatar name="John Doe" size="lg" showInPill />
+            <Avatar name="John Doe" size="md" showInPill />
+            <Avatar name="John Doe" size="default" showInPill />
+            <Avatar name="John Doe" size="sm" showInPill />
+        </GridList>
+        <GridList>
+            <Avatar name="John Doe" size="lg" imageUrl={IMAGE_URL} showInPill />
+            <Avatar name="John Doe" size="md" imageUrl={IMAGE_URL} showInPill />
+            <Avatar name="John Doe" size="default" imageUrl={IMAGE_URL} showInPill />
+            <Avatar name="John Doe" size="sm" imageUrl={IMAGE_URL} showInPill />
+        </GridList>
+    </GridList>
+);
+
+export const outlined = () => (
+    <GridList>
+        <Avatar name="John Doe" size="lg" imageUrl={IMAGE_URL} isOutlined />
+        <Avatar name="John Doe" size="lg" showInPill imageUrl={IMAGE_URL} isOutlined />
+    </GridList>
+);
+
+export const withOnClick = () => (
+    <GridList>
+        <Avatar name="John Doe" onClick={() => window.alert('Avatar clicked')} />
+        <Avatar name="John Doe" onClick={() => window.alert('Avatar clicked')} showInPill />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.tsx b/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.tsx
new file mode 100644
index 0000000000000..9e5ec025e08e3
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.tsx
@@ -0,0 +1,40 @@
+import React, { useState } from 'react';
+import { AvatarImage, AvatarImageWrapper, AvatarText, Container } from './components';
+import { AvatarProps } from './types';
+import getAvatarColor, { getNameInitials } from './utils';
+
+export const avatarDefaults: AvatarProps = {
+    name: 'User name',
+    size: 'default',
+    showInPill: false,
+    isOutlined: false,
+};
+
+export const Avatar = ({
+    name = avatarDefaults.name,
+    imageUrl,
+    size = avatarDefaults.size,
+    onClick,
+    showInPill = avatarDefaults.showInPill,
+    isOutlined = avatarDefaults.isOutlined,
+}: AvatarProps) => {
+    const [hasError, setHasError] = useState(false);
+
+    return (
+        <Container onClick={onClick} $hasOnClick={!!onClick} $showInPill={showInPill}>
+            <AvatarImageWrapper
+                $color={getAvatarColor(name)}
+                $size={size}
+                $isOutlined={isOutlined}
+                $hasImage={!!imageUrl}
+            >
+                {!hasError && imageUrl ? (
+                    <AvatarImage src={imageUrl} onError={() => setHasError(true)} />
+                ) : (
+                    <>{getNameInitials(name)} </>
+                )}
+            </AvatarImageWrapper>
+            {showInPill && <AvatarText $size={size}>{name}</AvatarText>}
+        </Container>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/_tests_/getNameInitials.test.ts b/datahub-web-react/src/alchemy-components/components/Avatar/_tests_/getNameInitials.test.ts
new file mode 100644
index 0000000000000..54bb258acb0d8
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Avatar/_tests_/getNameInitials.test.ts
@@ -0,0 +1,34 @@
+import { getNameInitials } from '../utils';
+
+describe('get initials of the name', () => {
+    it('get initials of name with first name and last name', () => {
+        expect(getNameInitials('John Doe ')).toEqual('JD');
+    });
+    it('get initials of name with first name and last name in lower case', () => {
+        expect(getNameInitials('john doe')).toEqual('JD');
+    });
+    it('get initials of name with only first name', () => {
+        expect(getNameInitials('Robert')).toEqual('RO');
+    });
+    it('get initials of name with only first name in lower case', () => {
+        expect(getNameInitials('robert')).toEqual('RO');
+    });
+    it('get initials of name with three names', () => {
+        expect(getNameInitials('James Edward Brown')).toEqual('JB');
+    });
+    it('get initials of name with four names', () => {
+        expect(getNameInitials('Michael James Alexander Scott')).toEqual('MS');
+    });
+    it('get initials of name with a hyphen', () => {
+        expect(getNameInitials('Mary-Jane Watson')).toEqual('MW');
+    });
+    it('get initials of name with an apostrophe', () => {
+        expect(getNameInitials("O'Connor")).toEqual('OC');
+    });
+    it('get initials of name with a single letter', () => {
+        expect(getNameInitials('J')).toEqual('J');
+    });
+    it('get initials of name with an empty string', () => {
+        expect(getNameInitials('')).toEqual('');
+    });
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/components.ts b/datahub-web-react/src/alchemy-components/components/Avatar/components.ts
new file mode 100644
index 0000000000000..bcd23a8ab086c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Avatar/components.ts
@@ -0,0 +1,51 @@
+import { colors } from '@src/alchemy-components/theme';
+import { AvatarSizeOptions } from '@src/alchemy-components/theme/config';
+import styled from 'styled-components';
+import { getAvatarColorStyles, getAvatarNameSizes, getAvatarSizes } from './utils';
+
+export const Container = styled.div<{ $hasOnClick: boolean; $showInPill?: boolean }>`
+    display: inline-flex;
+    align-items: center;
+    gap: 4px;
+    border-radius: 20px;
+    border: ${(props) => props.$showInPill && `1px solid ${colors.gray[100]}`};
+    padding: ${(props) => props.$showInPill && '3px 6px 3px 4px'};
+
+    ${(props) =>
+        props.$hasOnClick &&
+        `
+        :hover {
+        cursor: pointer;
+    }
+        `}
+`;
+
+export const AvatarImageWrapper = styled.div<{
+    $color: string;
+    $size?: AvatarSizeOptions;
+    $isOutlined?: boolean;
+    $hasImage?: boolean;
+}>`
+    ${(props) => getAvatarSizes(props.$size)}
+
+    border-radius: 50%;
+    color: ${(props) => props.$color};
+    border: ${(props) => props.$isOutlined && `1px solid ${colors.gray[1800]}`};
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    ${(props) => !props.$hasImage && getAvatarColorStyles(props.$color)}
+`;
+
+export const AvatarImage = styled.img`
+    width: 100%;
+    height: 100%;
+    object-fit: cover;
+    border-radius: 50%;
+`;
+
+export const AvatarText = styled.span<{ $size?: AvatarSizeOptions }>`
+    color: ${colors.gray[1700]};
+    font-weight: 600;
+    font-size: ${(props) => getAvatarNameSizes(props.$size)};
+`;
diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/index.ts b/datahub-web-react/src/alchemy-components/components/Avatar/index.ts
new file mode 100644
index 0000000000000..d3fb6dfa7c09e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Avatar/index.ts
@@ -0,0 +1 @@
+export { Avatar } from './Avatar';
diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/types.ts b/datahub-web-react/src/alchemy-components/components/Avatar/types.ts
new file mode 100644
index 0000000000000..98c554b620dcb
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Avatar/types.ts
@@ -0,0 +1,10 @@
+import { AvatarSizeOptions } from '@src/alchemy-components/theme/config';
+
+export interface AvatarProps {
+    name: string;
+    imageUrl?: string;
+    onClick?: () => void;
+    size?: AvatarSizeOptions;
+    showInPill?: boolean;
+    isOutlined?: boolean;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/utils.ts b/datahub-web-react/src/alchemy-components/components/Avatar/utils.ts
new file mode 100644
index 0000000000000..46b2ee25488b8
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Avatar/utils.ts
@@ -0,0 +1,64 @@
+import { colors } from '@src/alchemy-components/theme';
+
+export const getNameInitials = (userName: string) => {
+    if (!userName) return '';
+    const names = userName.trim().split(/[\s']+/); // Split by spaces or apostrophes
+    if (names.length === 1) {
+        const firstName = names[0];
+        return firstName.length > 1 ? firstName[0]?.toUpperCase() + firstName[1]?.toUpperCase() : firstName[0];
+    }
+    return names[0][0]?.toUpperCase() + names[names.length - 1][0]?.toUpperCase() || '';
+};
+
+export function hashString(str: string) {
+    let hash = 0;
+    if (str.length === 0) {
+        return hash;
+    }
+    for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i);
+        // eslint-disable-next-line
+        hash = (hash << 5) - hash + char;
+        // eslint-disable-next-line
+        hash = hash & hash; // Convert to 32bit integer
+    }
+    return Math.abs(hash);
+}
+
+const colorMap = {
+    [colors.violet[500]]: { backgroundColor: colors.gray[1000], border: `1px solid ${colors.violet[1000]}` },
+    [colors.blue[1000]]: { backgroundColor: colors.gray[1100], border: `1px solid ${colors.blue[200]}` },
+    [colors.gray[600]]: { backgroundColor: colors.gray[1500], border: `1px solid ${colors.gray[100]}` },
+};
+
+const avatarColors = Object.keys(colorMap);
+
+export const getAvatarColorStyles = (color) => {
+    return {
+        ...colorMap[color],
+    };
+};
+
+export default function getAvatarColor(name: string) {
+    return avatarColors[hashString(name) % avatarColors.length];
+}
+
+export const getAvatarSizes = (size) => {
+    const sizeMap = {
+        sm: { width: '18px', height: '18px', fontSize: '8px' },
+        md: { width: '24px', height: '24px', fontSize: '12px' },
+        lg: { width: '28px', height: '28px', fontSize: '14px' },
+        default: { width: '20px', height: '20px', fontSize: '10px' },
+    };
+
+    return {
+        ...sizeMap[size],
+    };
+};
+
+export const getAvatarNameSizes = (size) => {
+    if (size === 'lg') return '16px';
+    if (size === 'sm') return '10px';
+    if (size === 'md') return '14px';
+    return '12px';
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Badge/Badge.stories.tsx b/datahub-web-react/src/alchemy-components/components/Badge/Badge.stories.tsx
new file mode 100644
index 0000000000000..88d499226feaf
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Badge/Badge.stories.tsx
@@ -0,0 +1,102 @@
+import React from 'react';
+
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import type { Meta, StoryObj } from '@storybook/react';
+
+import { GridList } from '@components/.docs/mdx-components';
+import { Badge, badgeDefault } from './Badge';
+import pillMeta from '../Pills/Pill.stories';
+import { omitKeys } from './utils';
+
+const pillMetaArgTypes = omitKeys(pillMeta.argTypes, ['label']);
+const pillMetaArgs = omitKeys(pillMeta.args, ['label']);
+
+const meta = {
+    title: 'Components / Badge',
+    component: Badge,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.EXPERIMENTAL],
+        docs: {
+            subtitle: 'A component that is used to get badge',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        count: {
+            description: 'Count to show.',
+            table: {
+                defaultValue: { summary: `${badgeDefault.count}` },
+            },
+            control: {
+                type: 'number',
+            },
+        },
+        overflowCount: {
+            description: 'Max count to show.',
+            table: {
+                defaultValue: { summary: `${badgeDefault.overflowCount}` },
+            },
+            control: {
+                type: 'number',
+            },
+        },
+        showZero: {
+            description: 'Whether to show badge when `count` is zero.',
+            table: {
+                defaultValue: { summary: `${badgeDefault.showZero}` },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        ...pillMetaArgTypes,
+    },
+
+    // Define defaults
+    args: {
+        count: 100,
+        overflowCount: badgeDefault.overflowCount,
+        showZero: badgeDefault.showZero,
+        ...pillMetaArgs,
+    },
+} satisfies Meta<typeof Badge>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Badge {...props} />,
+};
+
+export const sizes = () => (
+    <GridList>
+        <Badge count={100} />
+        <Badge count={100} size="sm" />
+        <Badge count={100} size="lg" />
+    </GridList>
+);
+
+export const colors = () => (
+    <GridList>
+        <Badge count={100} />
+        <Badge count={100} colorScheme="violet" />
+        <Badge count={100} colorScheme="green" />
+        <Badge count={100} colorScheme="red" />
+        <Badge count={100} colorScheme="blue" />
+        <Badge count={100} colorScheme="gray" />
+    </GridList>
+);
+
+export const withIcon = () => (
+    <GridList>
+        <Badge count={100} leftIcon="AutoMode" />
+        <Badge count={100} rightIcon="Close" />
+        <Badge count={100} leftIcon="AutoMode" rightIcon="Close" />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Badge/Badge.tsx b/datahub-web-react/src/alchemy-components/components/Badge/Badge.tsx
new file mode 100644
index 0000000000000..1c934ef120eee
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Badge/Badge.tsx
@@ -0,0 +1,29 @@
+import { Pill } from '@components';
+import React, { useMemo } from 'react';
+
+import { BadgeProps } from './types';
+import { formatBadgeValue } from './utils';
+import { BadgeContainer } from './components';
+
+export const badgeDefault: BadgeProps = {
+    count: 0,
+    overflowCount: 99,
+    showZero: false,
+};
+
+export function Badge({
+    count = badgeDefault.count,
+    overflowCount = badgeDefault.overflowCount,
+    showZero = badgeDefault.showZero,
+    ...props
+}: BadgeProps) {
+    const label = useMemo(() => formatBadgeValue(count, overflowCount), [count, overflowCount]);
+
+    if (!showZero && count === 0) return null;
+
+    return (
+        <BadgeContainer title={`${count}`}>
+            <Pill label={label} {...props} />
+        </BadgeContainer>
+    );
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Badge/components.ts b/datahub-web-react/src/alchemy-components/components/Badge/components.ts
new file mode 100644
index 0000000000000..a7791cd4f5ff8
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Badge/components.ts
@@ -0,0 +1,6 @@
+import styled from 'styled-components';
+
+export const BadgeContainer = styled.div({
+    // Base root styles
+    display: 'inline-flex',
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Badge/index.ts b/datahub-web-react/src/alchemy-components/components/Badge/index.ts
new file mode 100644
index 0000000000000..26a9e305c7ffd
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Badge/index.ts
@@ -0,0 +1 @@
+export { Badge } from './Badge';
diff --git a/datahub-web-react/src/alchemy-components/components/Badge/types.ts b/datahub-web-react/src/alchemy-components/components/Badge/types.ts
new file mode 100644
index 0000000000000..21348f2a08341
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Badge/types.ts
@@ -0,0 +1,8 @@
+import { HTMLAttributes } from 'react';
+import { PillProps } from '../Pills/types';
+
+export interface BadgeProps extends HTMLAttributes<HTMLElement>, Omit<PillProps, 'label'> {
+    count: number;
+    overflowCount?: number;
+    showZero?: boolean;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Badge/utils.ts b/datahub-web-react/src/alchemy-components/components/Badge/utils.ts
new file mode 100644
index 0000000000000..e59ec2af998e7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Badge/utils.ts
@@ -0,0 +1,15 @@
+export const formatBadgeValue = (value: number, overflowCount?: number): string => {
+    if (overflowCount === undefined || value < overflowCount) return String(value);
+
+    return `${overflowCount}+`;
+};
+
+export function omitKeys<T extends object, K extends keyof T>(obj: T, keys: K[]): Omit<T, K> {
+    const { ...rest } = obj;
+
+    keys.forEach((key) => {
+        delete rest[key];
+    });
+
+    return rest;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.stories.tsx b/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.stories.tsx
new file mode 100644
index 0000000000000..1258ff398c0a7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.stories.tsx
@@ -0,0 +1,90 @@
+import React from 'react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import type { Meta, StoryObj } from '@storybook/react';
+import { BarChart } from './BarChart';
+import { getMockedProps } from './utils';
+
+const meta = {
+    title: 'Charts / BarChart',
+    component: BarChart,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.EXPERIMENTAL],
+        docs: {
+            subtitle: 'A component that is used to show BarChart',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        data: {
+            description: 'Array of datum to show',
+        },
+        xAccessor: {
+            description: 'A function to convert datum to value of X',
+        },
+        yAccessor: {
+            description: 'A function to convert datum to value of Y',
+        },
+        renderTooltipContent: {
+            description: 'A function to replace default rendering of toolbar',
+        },
+        margin: {
+            description: 'Add margins to chart',
+        },
+        leftAxisTickFormat: {
+            description: 'A function to format labels of left axis',
+        },
+        leftAxisTickLabelProps: {
+            description: 'Props for label of left axis',
+        },
+        bottomAxisTickFormat: {
+            description: 'A function to format labels of bottom axis',
+        },
+        bottomAxisTickLabelProps: {
+            description: 'Props for label of bottom axis',
+        },
+        barColor: {
+            description: 'Color of bar',
+            control: {
+                type: 'color',
+            },
+        },
+        barSelectedColor: {
+            description: 'Color of selected bar',
+            control: {
+                type: 'color',
+            },
+        },
+        gridColor: {
+            description: "Color of grid's lines",
+            control: {
+                type: 'color',
+            },
+        },
+        renderGradients: {
+            description: 'A function to render different gradients that can be used as colors',
+        },
+    },
+
+    // Define defaults
+    args: {
+        ...getMockedProps(),
+        renderTooltipContent: (datum) => <>DATUM: {JSON.stringify(datum)}</>,
+    },
+} satisfies Meta<typeof BarChart>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => (
+        <div style={{ width: '900px', height: '350px' }}>
+            <BarChart {...props} />
+        </div>
+    ),
+};
diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.tsx b/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.tsx
new file mode 100644
index 0000000000000..eb5465a1d1217
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.tsx
@@ -0,0 +1,152 @@
+import React, { useState } from 'react';
+import { colors } from '@src/alchemy-components/theme';
+import { TickLabelProps } from '@visx/axis';
+import { LinearGradient } from '@visx/gradient';
+import { ParentSize } from '@visx/responsive';
+import { Axis, AxisScale, BarSeries, Grid, Tooltip, XYChart } from '@visx/xychart';
+import dayjs from 'dayjs';
+import { Popover } from '../Popover';
+import { ChartWrapper, StyledBarSeries } from './components';
+import { BarChartProps } from './types';
+import { abbreviateNumber } from '../dataviz/utils';
+
+const commonTickLabelProps: TickLabelProps<any> = {
+    fontSize: 10,
+    fontFamily: 'Mulish',
+    fill: colors.gray[1700],
+};
+
+export const barChartDefault: BarChartProps<any> = {
+    data: [],
+    xAccessor: (datum) => datum?.x,
+    yAccessor: (datum) => datum?.y,
+    leftAxisTickFormat: abbreviateNumber,
+    leftAxisTickLabelProps: {
+        ...commonTickLabelProps,
+        textAnchor: 'end',
+    },
+    bottomAxisTickFormat: (value) => dayjs(value).format('DD MMM'),
+    bottomAxisTickLabelProps: {
+        ...commonTickLabelProps,
+        textAnchor: 'middle',
+        verticalAnchor: 'start',
+        width: 20,
+    },
+    barColor: 'url(#bar-gradient)',
+    barSelectedColor: colors.violet[500],
+    gridColor: '#e0e0e0',
+    renderGradients: () => <LinearGradient id="bar-gradient" from={colors.violet[500]} to="#917FFF" toOpacity={0.6} />,
+};
+
+export function BarChart<DatumType extends object = any>({
+    data,
+    xAccessor = barChartDefault.xAccessor,
+    yAccessor = barChartDefault.yAccessor,
+    renderTooltipContent,
+    margin,
+    leftAxisTickFormat = barChartDefault.leftAxisTickFormat,
+    leftAxisTickLabelProps = barChartDefault.leftAxisTickLabelProps,
+    bottomAxisTickFormat = barChartDefault.bottomAxisTickFormat,
+    bottomAxisTickLabelProps = barChartDefault.bottomAxisTickLabelProps,
+    barColor = barChartDefault.barColor,
+    barSelectedColor = barChartDefault.barSelectedColor,
+    gridColor = barChartDefault.gridColor,
+    renderGradients = barChartDefault.renderGradients,
+}: BarChartProps<DatumType>) {
+    const [hasSelectedBar, setHasSelectedBar] = useState<boolean>(false);
+
+    // FYI: additional margins to show left and bottom axises
+    const internalMargin = {
+        top: (margin?.top ?? 0) + 30,
+        right: margin?.right ?? 0,
+        bottom: (margin?.bottom ?? 0) + 35,
+        left: (margin?.left ?? 0) + 40,
+    };
+
+    const accessors = { xAccessor, yAccessor };
+
+    return (
+        <ChartWrapper>
+            <ParentSize>
+                {({ width, height }) => {
+                    return (
+                        <XYChart
+                            width={width}
+                            height={height}
+                            xScale={{ type: 'band', paddingInner: 0.4, paddingOuter: 0.1 }}
+                            yScale={{ type: 'linear', nice: true, round: true }}
+                            margin={internalMargin}
+                            captureEvents={false}
+                        >
+                            {renderGradients?.()}
+
+                            <Axis
+                                orientation="left"
+                                hideAxisLine
+                                hideTicks
+                                tickFormat={leftAxisTickFormat}
+                                tickLabelProps={leftAxisTickLabelProps}
+                            />
+
+                            <Axis
+                                orientation="bottom"
+                                numTicks={data.length}
+                                tickFormat={bottomAxisTickFormat}
+                                tickLabelProps={bottomAxisTickLabelProps}
+                                hideAxisLine
+                                hideTicks
+                            />
+
+                            <line
+                                x1={internalMargin.left}
+                                x2={internalMargin.left}
+                                y1={0}
+                                y2={height - internalMargin.bottom}
+                                stroke={gridColor}
+                            />
+
+                            <Grid rows columns={false} stroke={gridColor} strokeWidth={1} lineStyle={{}} />
+
+                            <StyledBarSeries
+                                as={BarSeries<AxisScale, AxisScale, DatumType>}
+                                $hasSelectedItem={hasSelectedBar}
+                                $color={barColor}
+                                $selectedColor={barSelectedColor}
+                                dataKey="bar-seria-0"
+                                data={data}
+                                radius={4}
+                                radiusTop
+                                onBlur={() => setHasSelectedBar(false)}
+                                onFocus={() => setHasSelectedBar(true)}
+                                // Internally the library doesn't emmit these events if handlers are empty
+                                // They are requred to show/hide/move tooltip
+                                onPointerMove={() => null}
+                                onPointerUp={() => null}
+                                onPointerOut={() => null}
+                                {...accessors}
+                            />
+
+                            <Tooltip<DatumType>
+                                snapTooltipToDatumX
+                                snapTooltipToDatumY
+                                unstyled
+                                applyPositionStyle
+                                renderTooltip={({ tooltipData }) => {
+                                    return (
+                                        tooltipData?.nearestDatum && (
+                                            <Popover
+                                                open
+                                                placement="topLeft"
+                                                content={renderTooltipContent?.(tooltipData.nearestDatum.datum)}
+                                            />
+                                        )
+                                    );
+                                }}
+                            />
+                        </XYChart>
+                    );
+                }}
+            </ParentSize>
+        </ChartWrapper>
+    );
+}
diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/components.tsx b/datahub-web-react/src/alchemy-components/components/BarChart/components.tsx
new file mode 100644
index 0000000000000..aa8f1320ef21d
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/BarChart/components.tsx
@@ -0,0 +1,34 @@
+import { colors } from '@src/alchemy-components/theme';
+import { BarSeries } from '@visx/xychart';
+import styled from 'styled-components';
+
+export const ChartWrapper = styled.div`
+    width: 100%;
+    height: 100%;
+    position: relative;
+`;
+
+export const StyledBarSeries = styled(BarSeries)<{
+    $hasSelectedItem?: boolean;
+    $color?: string;
+    $selectedColor?: string;
+}>`
+    & {
+        cursor: pointer;
+
+        fill: ${(props) => (props.$hasSelectedItem ? props.$selectedColor : props.$color) || colors.violet[500]};
+        ${(props) => props.$hasSelectedItem && 'opacity: 0.3;'}
+
+        :hover {
+            fill: ${(props) => props.$selectedColor || colors.violet[500]};
+            filter: drop-shadow(0px -2px 5px rgba(33, 23, 95, 0.3));
+            opacity: 1;
+        }
+
+        :focus {
+            fill: ${(props) => props.$selectedColor || colors.violet[500]};
+            outline: none;
+            opacity: 1;
+        }
+    }
+`;
diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/index.ts b/datahub-web-react/src/alchemy-components/components/BarChart/index.ts
new file mode 100644
index 0000000000000..fdfc3f3ab44a8
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/BarChart/index.ts
@@ -0,0 +1 @@
+export { BarChart } from './BarChart';
diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/types.ts b/datahub-web-react/src/alchemy-components/components/BarChart/types.ts
new file mode 100644
index 0000000000000..5fd7e2e63e241
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/BarChart/types.ts
@@ -0,0 +1,18 @@
+import { TickFormatter, TickLabelProps } from '@visx/axis';
+import { Margin } from '@visx/xychart';
+
+export type BarChartProps<DatumType extends object> = {
+    data: DatumType[];
+    xAccessor: (datum: DatumType) => string | number;
+    yAccessor: (datum: DatumType) => number;
+    renderTooltipContent?: (datum: DatumType) => React.ReactNode;
+    margin?: Margin;
+    leftAxisTickFormat?: TickFormatter<DatumType>;
+    leftAxisTickLabelProps?: TickLabelProps<DatumType>;
+    bottomAxisTickFormat?: TickFormatter<DatumType>;
+    bottomAxisTickLabelProps?: TickLabelProps<DatumType>;
+    barColor?: string;
+    barSelectedColor?: string;
+    gridColor?: string;
+    renderGradients?: () => React.ReactNode;
+};
diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/utils.ts b/datahub-web-react/src/alchemy-components/components/BarChart/utils.ts
new file mode 100644
index 0000000000000..0b592da7f59b0
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/BarChart/utils.ts
@@ -0,0 +1,26 @@
+import dayjs from 'dayjs';
+
+export function generateMockData(length = 30, maxValue = 50_000, minValue = 0) {
+    return Array(length)
+        .fill(0)
+        .map((_, index) => {
+            const date = dayjs()
+                .startOf('day')
+                .add(index - length, 'days')
+                .toDate();
+            const value = Math.max(Math.random() * maxValue, minValue);
+
+            return {
+                x: date,
+                y: value,
+            };
+        });
+}
+
+export function getMockedProps() {
+    return {
+        data: generateMockData(),
+        xAccessor: (datum) => datum.x,
+        yAccessor: (datum) => Math.max(datum.y, 1000),
+    };
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Button/Button.stories.tsx b/datahub-web-react/src/alchemy-components/components/Button/Button.stories.tsx
new file mode 100644
index 0000000000000..e2d7c2852da51
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Button/Button.stories.tsx
@@ -0,0 +1,203 @@
+import React from 'react';
+
+import type { Meta, StoryObj } from '@storybook/react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+
+import { GridList } from '@components/.docs/mdx-components';
+import { AVAILABLE_ICONS } from '@components';
+
+import { Button, buttonDefaults } from '.';
+
+// Auto Docs
+const meta = {
+    title: 'Forms / Button',
+    component: Button,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle:
+                'Buttons are used to trigger an action or event, such as submitting a form, opening a dialog, canceling an action, or performing a delete operation.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        children: {
+            description: 'The content of the Button.',
+            control: {
+                type: 'text',
+            },
+        },
+        variant: {
+            description: 'The variant of the Button.',
+            options: ['filled', 'outline', 'text'],
+            table: {
+                defaultValue: { summary: buttonDefaults.variant },
+            },
+            control: {
+                type: 'radio',
+            },
+        },
+        color: {
+            description: 'The color of the Button.',
+            options: ['violet', 'green', 'red', 'blue', 'gray'],
+            table: {
+                defaultValue: { summary: buttonDefaults.color },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        size: {
+            description: 'The size of the Button.',
+            options: ['sm', 'md', 'lg', 'xl'],
+            table: {
+                defaultValue: { summary: buttonDefaults.size },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        icon: {
+            description: 'The icon to display in the Button.',
+            type: 'string',
+            options: AVAILABLE_ICONS,
+            table: {
+                defaultValue: { summary: 'undefined' },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        iconPosition: {
+            description: 'The position of the icon in the Button.',
+            options: ['left', 'right'],
+            table: {
+                defaultValue: { summary: buttonDefaults.iconPosition },
+            },
+            control: {
+                type: 'radio',
+            },
+        },
+        isCircle: {
+            description:
+                'Whether the Button should be a circle. If this is selected, the Button will ignore children content, so add an Icon to the Button.',
+            table: {
+                defaultValue: { summary: buttonDefaults?.isCircle?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isLoading: {
+            description: 'Whether the Button is in a loading state.',
+            table: {
+                defaultValue: { summary: buttonDefaults?.isLoading?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isDisabled: {
+            description: 'Whether the Button is disabled.',
+            table: {
+                defaultValue: { summary: buttonDefaults?.isDisabled?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isActive: {
+            description: 'Whether the Button is active.',
+            table: {
+                defaultValue: { summary: buttonDefaults?.isActive?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        onClick: {
+            description: 'Function to call when the button is clicked',
+            table: {
+                defaultValue: { summary: 'undefined' },
+            },
+            action: 'clicked',
+        },
+    },
+
+    // Define defaults
+    args: {
+        children: 'Button Content',
+        variant: buttonDefaults.variant,
+        color: buttonDefaults.color,
+        size: buttonDefaults.size,
+        icon: undefined,
+        iconPosition: buttonDefaults.iconPosition,
+        isCircle: buttonDefaults.isCircle,
+        isLoading: buttonDefaults.isLoading,
+        isDisabled: buttonDefaults.isDisabled,
+        isActive: buttonDefaults.isActive,
+        onClick: () => console.log('Button clicked'),
+    },
+} satisfies Meta<typeof Button>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+// Basic story is what is displayed 1st in storybook & is used as the code sandbox
+// Pass props to this so that it can be customized via the UI props panel
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Button {...props}>Button</Button>,
+};
+
+export const states = () => (
+    <GridList>
+        <Button>Default</Button>
+        <Button isLoading>Loading State</Button>
+        <Button isActive>Active/Focus State</Button>
+        <Button isDisabled>Disabled State</Button>
+    </GridList>
+);
+
+export const colors = () => (
+    <GridList>
+        <Button>Violet Button</Button>
+        <Button color="green">Green Button</Button>
+        <Button color="red">Red Button</Button>
+        <Button color="blue">Blue Button</Button>
+        <Button color="gray">Gray Button</Button>
+    </GridList>
+);
+
+export const sizes = () => (
+    <GridList>
+        <Button size="sm">Small Button</Button>
+        <Button size="md">Regular Button</Button>
+        <Button size="lg">Large Button</Button>
+        <Button size="xl">XLarge Button</Button>
+    </GridList>
+);
+
+export const withIcon = () => (
+    <GridList>
+        <Button icon="Add">Icon Left</Button>
+        <Button icon="Add" iconPosition="right">
+            Icon Right
+        </Button>
+    </GridList>
+);
+
+export const circleShape = () => (
+    <GridList>
+        <Button icon="Add" size="sm" isCircle />
+        <Button icon="Add" isCircle />
+        <Button icon="Add" size="lg" isCircle />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Button/Button.tsx b/datahub-web-react/src/alchemy-components/components/Button/Button.tsx
new file mode 100644
index 0000000000000..a727b0faf97a9
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Button/Button.tsx
@@ -0,0 +1,60 @@
+import React from 'react';
+
+import { LoadingOutlined } from '@ant-design/icons';
+
+import { Icon } from '@components';
+
+import { ButtonBase } from './components';
+import { ButtonProps } from './types';
+
+export const buttonDefaults: ButtonProps = {
+    variant: 'filled',
+    color: 'violet',
+    size: 'md',
+    iconPosition: 'left',
+    isCircle: false,
+    isLoading: false,
+    isDisabled: false,
+    isActive: false,
+};
+
+export const Button = ({
+    variant = buttonDefaults.variant,
+    color = buttonDefaults.color,
+    size = buttonDefaults.size,
+    icon, // default undefined
+    iconPosition = buttonDefaults.iconPosition,
+    isCircle = buttonDefaults.isCircle,
+    isLoading = buttonDefaults.isLoading,
+    isDisabled = buttonDefaults.isDisabled,
+    isActive = buttonDefaults.isActive,
+    children,
+    ...props
+}: ButtonProps) => {
+    const sharedProps = {
+        variant,
+        color,
+        size,
+        isCircle,
+        isLoading,
+        isActive,
+        isDisabled,
+        disabled: isDisabled,
+    };
+
+    if (isLoading) {
+        return (
+            <ButtonBase {...sharedProps} {...props}>
+                <LoadingOutlined rotate={10} /> {!isCircle && children}
+            </ButtonBase>
+        );
+    }
+
+    return (
+        <ButtonBase {...sharedProps} {...props}>
+            {icon && iconPosition === 'left' && <Icon icon={icon} size={size} />}
+            {!isCircle && children}
+            {icon && iconPosition === 'right' && <Icon icon={icon} size={size} />}
+        </ButtonBase>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Button/components.ts b/datahub-web-react/src/alchemy-components/components/Button/components.ts
new file mode 100644
index 0000000000000..49fa9a12ede6e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Button/components.ts
@@ -0,0 +1,27 @@
+import styled from 'styled-components';
+
+import { spacing } from '@components/theme';
+import { ButtonProps } from './types';
+import { getButtonStyle } from './utils';
+
+export const ButtonBase = styled.button(
+    // Dynamic styles
+    (props: ButtonProps) => ({ ...getButtonStyle(props as ButtonProps) }),
+    {
+        // Base root styles
+        display: 'flex',
+        alignItems: 'center',
+        gap: spacing.xsm,
+        cursor: 'pointer',
+        transition: `all 0.15s ease`,
+
+        // For transitions between focus/active and hover states
+        outlineColor: 'transparent',
+        outlineStyle: 'solid',
+
+        // Base Disabled styles
+        '&:disabled': {
+            cursor: 'not-allowed',
+        },
+    },
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Button/index.ts b/datahub-web-react/src/alchemy-components/components/Button/index.ts
new file mode 100644
index 0000000000000..745d8377f9fbb
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Button/index.ts
@@ -0,0 +1,2 @@
+export { Button, buttonDefaults } from './Button';
+export type { ButtonProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Button/types.ts b/datahub-web-react/src/alchemy-components/components/Button/types.ts
new file mode 100644
index 0000000000000..f510ff4c6c13c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Button/types.ts
@@ -0,0 +1,16 @@
+import { ButtonHTMLAttributes } from 'react';
+
+import type { IconNames } from '@components';
+import type { SizeOptions, ColorOptions } from '@components/theme/config';
+
+export interface ButtonProps extends ButtonHTMLAttributes<HTMLButtonElement> {
+    variant?: 'filled' | 'outline' | 'text';
+    color?: ColorOptions;
+    size?: SizeOptions;
+    icon?: IconNames;
+    iconPosition?: 'left' | 'right';
+    isCircle?: boolean;
+    isLoading?: boolean;
+    isDisabled?: boolean;
+    isActive?: boolean;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Button/utils.ts b/datahub-web-react/src/alchemy-components/components/Button/utils.ts
new file mode 100644
index 0000000000000..c08f4f067304d
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Button/utils.ts
@@ -0,0 +1,238 @@
+/*
+ * Button Style Utilities
+ */
+
+import { typography, colors, shadows, radius, spacing } from '@components/theme';
+import { getColor, getFontSize } from '@components/theme/utils';
+import { ButtonProps } from './types';
+
+// Utility function to get color styles for button - does not generate CSS
+const getButtonColorStyles = (variant, color) => {
+    const color500 = getColor(color, 500); // value of 500 shade
+    const isViolet = color === 'violet';
+
+    const base = {
+        // Backgrounds
+        bgColor: color500,
+        hoverBgColor: getColor(color, 600),
+        activeBgColor: getColor(color, 700),
+        disabledBgColor: getColor('gray', 100),
+
+        // Borders
+        borderColor: color500,
+        activeBorderColor: getColor(color, 300),
+        disabledBorderColor: getColor('gray', 200),
+
+        // Text
+        textColor: colors.white,
+        disabledTextColor: getColor('gray', 300),
+    };
+
+    // Specific color override for white
+    if (color === 'white') {
+        base.textColor = colors.black;
+        base.disabledTextColor = getColor('gray', 500);
+    }
+
+    // Specific color override for gray
+    if (color === 'gray') {
+        base.textColor = getColor('gray', 500);
+        base.bgColor = getColor('gray', 100);
+        base.borderColor = getColor('gray', 100);
+
+        base.hoverBgColor = getColor('gray', 100);
+        base.activeBgColor = getColor('gray', 200);
+    }
+
+    // Override styles for outline variant
+    if (variant === 'outline') {
+        return {
+            ...base,
+            bgColor: colors.transparent,
+            borderColor: color500,
+            textColor: color500,
+
+            hoverBgColor: isViolet ? getColor(color, 100) : getColor(color, 100),
+            activeBgColor: isViolet ? getColor(color, 100) : getColor(color, 200),
+
+            disabledBgColor: 'transparent',
+        };
+    }
+
+    // Override styles for text variant
+    if (variant === 'text') {
+        return {
+            ...base,
+            textColor: color500,
+
+            bgColor: colors.transparent,
+            borderColor: colors.transparent,
+            hoverBgColor: colors.transparent,
+            activeBgColor: colors.transparent,
+            disabledBgColor: colors.transparent,
+            disabledBorderColor: colors.transparent,
+        };
+    }
+
+    // Filled variable is the base style
+    return base;
+};
+
+// Generate color styles for button
+const getButtonVariantStyles = (variant, color) => {
+    const variantStyles = {
+        filled: {
+            backgroundColor: color.bgColor,
+            border: `1px solid ${color.borderColor}`,
+            color: color.textColor,
+            '&:hover': {
+                backgroundColor: color.hoverBgColor,
+                border: `1px solid ${color.hoverBgColor}`,
+                boxShadow: shadows.sm,
+            },
+            '&:disabled': {
+                backgroundColor: color.disabledBgColor,
+                border: `1px solid ${color.disabledBorderColor}`,
+                color: color.disabledTextColor,
+                boxShadow: shadows.xs,
+            },
+        },
+        outline: {
+            backgroundColor: 'transparent',
+            border: `1px solid ${color.borderColor}`,
+            color: color.textColor,
+            '&:hover': {
+                backgroundColor: color.hoverBgColor,
+                boxShadow: 'none',
+            },
+            '&:disabled': {
+                backgroundColor: color.disabledBgColor,
+                border: `1px solid ${color.disabledBorderColor}`,
+                color: color.disabledTextColor,
+                boxShadow: shadows.xs,
+            },
+        },
+        text: {
+            backgroundColor: 'transparent',
+            border: 'none',
+            color: color.textColor,
+            '&:hover': {
+                backgroundColor: color.hoverBgColor,
+            },
+            '&:disabled': {
+                backgroundColor: color.disabledBgColor,
+                color: color.disabledTextColor,
+            },
+        },
+    };
+
+    return variantStyles[variant];
+};
+
+// Generate font styles for button
+const getButtonFontStyles = (size) => {
+    const baseFontStyles = {
+        fontFamily: typography.fonts.body,
+        fontWeight: typography.fontWeights.normal,
+        lineHeight: typography.lineHeights.none,
+    };
+
+    const sizeStyles = {
+        sm: {
+            ...baseFontStyles,
+            fontSize: getFontSize(size), // 12px
+        },
+        md: {
+            ...baseFontStyles,
+            fontSize: getFontSize(size), // 14px
+        },
+        lg: {
+            ...baseFontStyles,
+            fontSize: getFontSize(size), // 16px
+        },
+        xl: {
+            ...baseFontStyles,
+            fontSize: getFontSize(size), // 18px
+        },
+    };
+
+    return sizeStyles[size];
+};
+
+// Generate radii styles for button
+const getButtonRadiiStyles = (isCircle) => {
+    if (isCircle) return { borderRadius: radius.full };
+    return { borderRadius: radius.sm }; // radius is the same for all button sizes
+};
+
+// Generate padding styles for button
+const getButtonPadding = (size, isCircle) => {
+    if (isCircle) return { padding: spacing.xsm };
+
+    const paddingStyles = {
+        sm: {
+            padding: '8px 12px',
+        },
+        md: {
+            padding: '10px 12px',
+        },
+        lg: {
+            padding: '10px 16px',
+        },
+        xl: {
+            padding: '12px 20px',
+        },
+    };
+
+    return paddingStyles[size];
+};
+
+// Generate active styles for button
+const getButtonActiveStyles = (styleColors) => ({
+    borderColor: 'transparent',
+    backgroundColor: styleColors.activeBgColor,
+    // TODO: Figure out how to make the #fff interior border transparent
+    boxShadow: `0 0 0 2px #fff, 0 0 0 4px ${styleColors.activeBgColor}`,
+});
+
+// Generate loading styles for button
+const getButtonLoadingStyles = () => ({
+    pointerEvents: 'none',
+    opacity: 0.75,
+});
+
+/*
+ * Main function to generate styles for button
+ */
+export const getButtonStyle = (props: ButtonProps) => {
+    const { variant, color, size, isCircle, isActive, isLoading } = props;
+
+    // Get map of colors
+    const colorStyles = getButtonColorStyles(variant, color) || ({} as any);
+
+    // Define styles for button
+    const variantStyles = getButtonVariantStyles(variant, colorStyles);
+    const fontStyles = getButtonFontStyles(size);
+    const radiiStyles = getButtonRadiiStyles(isCircle);
+    const paddingStyles = getButtonPadding(size, isCircle);
+
+    // Base of all generated styles
+    let styles = {
+        ...variantStyles,
+        ...fontStyles,
+        ...radiiStyles,
+        ...paddingStyles,
+    };
+
+    // Focus & Active styles are the same, but active styles are applied conditionally & override prevs styles
+    const activeStyles = { ...getButtonActiveStyles(colorStyles) };
+    styles['&:focus'] = activeStyles;
+    styles['&:active'] = activeStyles;
+    if (isActive) styles = { ...styles, ...activeStyles };
+
+    // Loading styles
+    if (isLoading) styles = { ...styles, ...getButtonLoadingStyles() };
+
+    // Return generated styles
+    return styles;
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Card/Card.stories.tsx b/datahub-web-react/src/alchemy-components/components/Card/Card.stories.tsx
new file mode 100644
index 0000000000000..336831fd15cfa
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Card/Card.stories.tsx
@@ -0,0 +1,141 @@
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import { GridList } from '@src/alchemy-components/.docs/mdx-components';
+import { colors } from '@src/alchemy-components/theme';
+import type { Meta, StoryObj } from '@storybook/react';
+import React from 'react';
+import { Card, cardDefaults } from '.';
+import { Icon } from '../Icon';
+
+// Auto Docs
+const meta = {
+    title: 'Components / Card',
+    component: Card,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'Used to render a card.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        title: {
+            description: 'The title of the card',
+            table: {
+                defaultValue: { summary: `${cardDefaults.title}` },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        subTitle: {
+            description: 'The subtitle of the card',
+            control: {
+                type: 'text',
+            },
+        },
+        icon: {
+            description: 'The icon on the card',
+            control: {
+                type: 'text',
+            },
+        },
+        iconAlignment: {
+            description: 'Whether the alignment of icon is horizontal or vertical',
+            table: {
+                defaultValue: { summary: `${cardDefaults.iconAlignment}` },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        percent: {
+            description: 'The percent value on the pill of the card',
+            control: {
+                type: 'number',
+            },
+        },
+        button: {
+            description: 'The button on the card',
+            control: {
+                type: 'text',
+            },
+        },
+        width: {
+            description: 'The width of the card',
+            control: {
+                type: 'text',
+            },
+        },
+        onClick: {
+            description: 'The on click function for the card',
+        },
+    },
+
+    // Define default args
+    args: {
+        title: 'Title',
+        subTitle: 'Subtitle',
+        iconAlignment: 'horizontal',
+        width: '150px',
+    },
+} satisfies Meta<typeof Card>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+// Basic story is what is displayed 1st in storybook
+// Pass props to this so that it can be customized via the UI props panel
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Card {...props} />,
+};
+
+export const withChildren = () => (
+    <Card title="Title" subTitle="Subtitle">
+        <div style={{ backgroundColor: colors.gray[1000], padding: '8px 32px' }}>Children of the card (Swap me)</div>
+    </Card>
+);
+
+export const withoutSubtitle = () => (
+    <Card title="Title">
+        <div style={{ backgroundColor: colors.gray[1000], padding: '8px 32px' }}>Children of the card (Swap me)</div>
+    </Card>
+);
+
+export const withIcon = () => (
+    <GridList>
+        <Card title="Title" subTitle="Subtitle" icon={<Icon icon="Cloud" color="gray" />} />
+        <Card title="Title" subTitle="Subtitle" icon={<Icon icon="Cloud" color="gray" />} iconAlignment="vertical" />
+    </GridList>
+);
+
+export const withButton = () => (
+    <Card
+        title="Title"
+        subTitle="Subtitle"
+        button={<Icon icon="Download" color="gray" size="2xl" />}
+        onClick={() => window.alert('Card clicked')}
+    />
+);
+
+export const withPercentPill = () => <Card title="Title" subTitle="Subtitle" percent={2} />;
+
+export const withAllTheElements = () => (
+    <Card
+        title="Title"
+        subTitle="Subtitle"
+        percent={2}
+        icon={<Icon icon="Cloud" color="gray" />}
+        button={<Icon icon="Download" color="gray" size="2xl" />}
+        onClick={() => window.alert('Card clicked')}
+    >
+        <div style={{ backgroundColor: colors.gray[1000], padding: '8px 32px' }}>Children of the card (Swap me)</div>
+    </Card>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Card/Card.tsx b/datahub-web-react/src/alchemy-components/components/Card/Card.tsx
new file mode 100644
index 0000000000000..55c581251bea9
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Card/Card.tsx
@@ -0,0 +1,48 @@
+import React from 'react';
+import { CardProps } from './types';
+import { CardContainer, Header, SubTitle, SubTitleContainer, Title, TitleContainer } from './components';
+import { Pill } from '../Pills';
+
+export const cardDefaults: CardProps = {
+    title: 'Title',
+    iconAlignment: 'horizontal',
+};
+
+export const Card = ({
+    title = cardDefaults.title,
+    iconAlignment = cardDefaults.iconAlignment,
+    subTitle,
+    percent,
+    button,
+    onClick,
+    icon,
+    children,
+    width,
+}: CardProps) => {
+    return (
+        <CardContainer hasButton={!!button} onClick={onClick} width={width}>
+            <Header iconAlignment={iconAlignment}>
+                {icon && <div>{icon}</div>}
+                <TitleContainer>
+                    <Title>
+                        {title}
+                        {!!percent && (
+                            <Pill
+                                label={`${Math.abs(percent)}%`}
+                                size="sm"
+                                colorScheme={percent < 0 ? 'red' : 'green'}
+                                leftIcon={percent < 0 ? 'TrendingDown' : 'TrendingUp'}
+                                clickable={false}
+                            />
+                        )}
+                    </Title>
+                    <SubTitleContainer>
+                        <SubTitle>{subTitle}</SubTitle>
+                        {button}
+                    </SubTitleContainer>
+                </TitleContainer>
+            </Header>
+            {children}
+        </CardContainer>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Card/components.ts b/datahub-web-react/src/alchemy-components/components/Card/components.ts
new file mode 100644
index 0000000000000..bb3821fffc7f5
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Card/components.ts
@@ -0,0 +1,59 @@
+import { colors, radius, spacing, typography } from '@src/alchemy-components/theme';
+import { IconAlignmentOptions } from '@src/alchemy-components/theme/config';
+import styled from 'styled-components';
+
+export const CardContainer = styled.div<{ hasButton: boolean; width?: string }>(({ hasButton, width }) => ({
+    border: `1px solid ${colors.gray[100]}`,
+    borderRadius: radius.lg,
+    padding: spacing.md,
+    minWidth: '150px',
+    boxShadow: '0px 1px 2px 0px rgba(33, 23, 95, 0.07)',
+    backgroundColor: colors.white,
+    display: 'flex',
+    flexDirection: 'column',
+    gap: spacing.md,
+    width,
+
+    '&:hover': hasButton
+        ? {
+              border: `1px solid ${colors.violet[500]}`,
+              cursor: 'pointer',
+          }
+        : {},
+}));
+
+export const Header = styled.div<{ iconAlignment?: IconAlignmentOptions }>(({ iconAlignment }) => ({
+    display: 'flex',
+    flexDirection: iconAlignment === 'horizontal' ? 'row' : 'column',
+    alignItems: iconAlignment === 'horizontal' ? 'center' : 'start',
+    gap: spacing.sm,
+    width: '100%',
+}));
+
+export const TitleContainer = styled.div({
+    display: 'flex',
+    flexDirection: 'column',
+    gap: 2,
+    width: '100%',
+});
+
+export const Title = styled.div({
+    fontSize: typography.fontSizes.lg,
+    fontWeight: typography.fontWeights.bold,
+    color: colors.gray[600],
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing.xsm,
+});
+
+export const SubTitleContainer = styled.div({
+    display: 'flex',
+    justifyContent: 'space-between',
+    alignItems: 'center',
+});
+
+export const SubTitle = styled.div({
+    fontSize: typography.fontSizes.md,
+    fontWeight: typography.fontWeights.normal,
+    color: colors.gray[1700],
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Card/index.ts b/datahub-web-react/src/alchemy-components/components/Card/index.ts
new file mode 100644
index 0000000000000..b0eed059aafd8
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Card/index.ts
@@ -0,0 +1,2 @@
+export { Card, cardDefaults } from './Card';
+export type { CardProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Card/types.ts b/datahub-web-react/src/alchemy-components/components/Card/types.ts
new file mode 100644
index 0000000000000..e5b0e36f83e4c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Card/types.ts
@@ -0,0 +1,13 @@
+import { IconAlignmentOptions } from '@src/alchemy-components/theme/config';
+
+export interface CardProps {
+    title: string;
+    subTitle?: string;
+    percent?: number;
+    button?: React.ReactNode;
+    onClick?: () => void;
+    icon?: React.ReactNode;
+    iconAlignment?: IconAlignmentOptions;
+    children?: React.ReactNode;
+    width?: string;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.stories.tsx b/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.stories.tsx
new file mode 100644
index 0000000000000..e546c2ea526cb
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.stories.tsx
@@ -0,0 +1,156 @@
+import React from 'react';
+import type { Meta, StoryObj } from '@storybook/react';
+import { GridList } from '@components/.docs/mdx-components';
+import { Checkbox, checkboxDefaults, CheckboxGroup } from './Checkbox';
+import { CheckboxProps } from './types';
+import { Heading } from '../Heading';
+
+const MOCK_CHECKBOXES: CheckboxProps[] = [
+    {
+        label: 'Label 1',
+        error: '',
+        isChecked: false,
+        isDisabled: false,
+        isIntermediate: false,
+        isRequired: false,
+    },
+    {
+        label: 'Label 2',
+        error: '',
+        isChecked: false,
+        isDisabled: false,
+        isIntermediate: false,
+        isRequired: false,
+    },
+    {
+        label: 'Label 3',
+        error: '',
+        isChecked: false,
+        isDisabled: false,
+        isIntermediate: false,
+        isRequired: false,
+    },
+];
+
+const meta = {
+    title: 'Forms / Checkbox',
+    component: Checkbox,
+    parameters: {
+        layout: 'centered',
+        docs: {
+            subtitle: 'A component that is used to get user input in the state of a check box.',
+        },
+    },
+    argTypes: {
+        label: {
+            description: 'Label for the Checkbox.',
+            table: {
+                defaultValue: { summary: checkboxDefaults.label },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        error: {
+            description: 'Enforce error state on the Checkbox.',
+            table: {
+                defaultValue: { summary: checkboxDefaults.error },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        isChecked: {
+            description: 'Whether the Checkbox is checked.',
+            table: {
+                defaultValue: { summary: checkboxDefaults?.isChecked?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isDisabled: {
+            description: 'Whether the Checkbox is in disabled state.',
+            table: {
+                defaultValue: { summary: checkboxDefaults?.isDisabled?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isIntermediate: {
+            description: 'Whether the Checkbox is in intermediate state.',
+            table: {
+                defaultValue: { summary: checkboxDefaults?.isIntermediate?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isRequired: {
+            description: 'Whether the Checkbox is a required field.',
+            table: {
+                defaultValue: { summary: checkboxDefaults?.isRequired?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+    },
+    args: {
+        label: checkboxDefaults.label,
+        error: checkboxDefaults.error,
+        isChecked: checkboxDefaults.isChecked,
+        isDisabled: checkboxDefaults.isDisabled,
+        isIntermediate: checkboxDefaults.isIntermediate,
+        isRequired: checkboxDefaults.isRequired,
+    },
+} satisfies Meta<typeof Checkbox>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Checkbox {...props} />,
+};
+
+export const states = () => (
+    <GridList>
+        <Checkbox label="Default" />
+        <Checkbox label="Checked" isChecked />
+        <Checkbox label="Error" isChecked error="Error" />
+        <Checkbox label="Required" isChecked isRequired />
+    </GridList>
+);
+
+export const intermediate = () => {
+    return (
+        <GridList>
+            <Checkbox label="Primary" isChecked isIntermediate />
+            <Checkbox label="Error" isChecked isIntermediate error="Error" />
+        </GridList>
+    );
+};
+
+export const disabledStates = () => (
+    <GridList>
+        <Checkbox label="Default" isDisabled />
+        <Checkbox label="Checked" isChecked isDisabled />
+        <Checkbox label="Intermediate" isChecked isDisabled isIntermediate />
+    </GridList>
+);
+
+export const checkboxGroups = () => (
+    <GridList isVertical>
+        <div>
+            <Heading>Horizontal Checkbox Group</Heading>
+            <CheckboxGroup checkboxes={MOCK_CHECKBOXES} />
+        </div>
+        <div>
+            <Heading>Vertical Checkbox Group</Heading>
+            <CheckboxGroup isVertical checkboxes={MOCK_CHECKBOXES} />
+        </div>
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.tsx b/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.tsx
new file mode 100644
index 0000000000000..6ab4db74610e4
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.tsx
@@ -0,0 +1,103 @@
+import React, { useEffect, useState } from 'react';
+import { CheckboxProps, CheckboxGroupProps } from './types';
+import {
+    CheckboxBase,
+    CheckboxContainer,
+    CheckboxGroupContainer,
+    Checkmark,
+    HoverState,
+    Label,
+    Required,
+    StyledCheckbox,
+} from './components';
+
+export const checkboxDefaults: CheckboxProps = {
+    label: 'Label',
+    error: '',
+    isChecked: false,
+    isDisabled: false,
+    isIntermediate: false,
+    isRequired: false,
+    setIsChecked: () => {},
+};
+
+export const Checkbox = ({
+    label = checkboxDefaults.label,
+    error = checkboxDefaults.error,
+    isChecked = checkboxDefaults.isChecked,
+    isDisabled = checkboxDefaults.isDisabled,
+    isIntermediate = checkboxDefaults.isIntermediate,
+    isRequired = checkboxDefaults.isRequired,
+    setIsChecked = checkboxDefaults.setIsChecked,
+    ...props
+}: CheckboxProps) => {
+    const [checked, setChecked] = useState(isChecked || false);
+    const [isHovering, setIsHovering] = useState(false);
+
+    useEffect(() => {
+        setChecked(isChecked || false);
+    }, [isChecked]);
+
+    const id = props.id || `checkbox-${label}`;
+
+    return (
+        <CheckboxContainer>
+            <Label aria-label={label}>
+                {label} {isRequired && <Required>*</Required>}
+            </Label>
+            <CheckboxBase
+                onClick={() => {
+                    if (!isDisabled) {
+                        setChecked(!checked);
+                        setIsChecked?.(!checked);
+                    }
+                }}
+            >
+                <StyledCheckbox
+                    type="checkbox"
+                    id="checked-input"
+                    checked={checked}
+                    disabled={isDisabled || false}
+                    error={error || ''}
+                    onChange={() => null}
+                    aria-labelledby={id}
+                    aria-checked={checked}
+                    {...props}
+                />
+                <Checkmark
+                    intermediate={isIntermediate || false}
+                    error={error || ''}
+                    disabled={isDisabled || false}
+                    checked={checked || false}
+                    onMouseOver={() => setIsHovering(true)}
+                    onMouseLeave={() => setIsHovering(false)}
+                />
+                <HoverState
+                    isHovering={!isDisabled ? isHovering : false}
+                    error={error || ''}
+                    checked={checked || false}
+                    disabled={isDisabled || false}
+                />
+            </CheckboxBase>
+        </CheckboxContainer>
+    );
+};
+
+export const CheckboxGroup = ({ isVertical, checkboxes }: CheckboxGroupProps) => {
+    if (!checkboxes.length) {
+        return <></>;
+    }
+
+    return (
+        <CheckboxGroupContainer isVertical={isVertical}>
+            {checkboxes.map((checkbox) => {
+                const props = { ...checkbox };
+                return (
+                    <React.Fragment key={checkbox.label}>
+                        <Checkbox {...props} />
+                    </React.Fragment>
+                );
+            })}
+        </CheckboxGroupContainer>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/components.ts b/datahub-web-react/src/alchemy-components/components/Checkbox/components.ts
new file mode 100644
index 0000000000000..6a4ad08c9c4ce
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Checkbox/components.ts
@@ -0,0 +1,91 @@
+import { borders, colors, spacing, transform, zIndices, radius } from '@components/theme';
+import styled from 'styled-components';
+import { getCheckboxColor, getCheckboxHoverBackgroundColor } from './utils';
+import { formLabelTextStyles } from '../commonStyles';
+
+export const CheckboxContainer = styled.div({
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+});
+
+export const Label = styled.div({
+    ...formLabelTextStyles,
+});
+
+export const Required = styled.span({
+    color: colors.red[500],
+    marginLeft: spacing.xxsm,
+});
+
+export const CheckboxBase = styled.div({
+    position: 'relative',
+    width: '30px',
+    height: '30px',
+});
+
+export const StyledCheckbox = styled.input<{
+    checked: boolean;
+    error: string;
+    disabled: boolean;
+}>(({ error, checked, disabled }) => ({
+    position: 'absolute',
+    opacity: 0,
+    height: 0,
+    width: 0,
+    '&:checked + div': {
+        backgroundColor: getCheckboxColor(checked, error, disabled, 'background'),
+    },
+    '&:checked + div:after': {
+        display: 'block',
+    },
+}));
+
+export const Checkmark = styled.div<{ intermediate?: boolean; error: string; checked: boolean; disabled: boolean }>(
+    ({ intermediate, checked, error, disabled }) => ({
+        position: 'absolute',
+        top: '4px',
+        left: '11px',
+        zIndex: zIndices.docked,
+        height: '18px',
+        width: '18px',
+        borderRadius: '3px',
+        border: `${borders['2px']} ${getCheckboxColor(checked, error, disabled, undefined)}`,
+        transition: 'all 0.2s ease-in-out',
+        cursor: 'pointer',
+        '&:after': {
+            content: '""',
+            position: 'absolute',
+            display: 'none',
+            left: !intermediate ? '6px' : '8px',
+            top: !intermediate ? '1px' : '3px',
+            width: !intermediate ? '5px' : '0px',
+            height: '10px',
+            border: 'solid white',
+            borderWidth: '0 3px 3px 0',
+            transform: !intermediate ? 'rotate(45deg)' : transform.rotate[90],
+        },
+    }),
+);
+
+export const HoverState = styled.div<{ isHovering: boolean; error: string; checked: boolean; disabled: boolean }>(
+    ({ isHovering, error, checked }) => ({
+        width: '40px',
+        height: '40px',
+        backgroundColor: !isHovering ? 'transparent' : getCheckboxHoverBackgroundColor(checked, error),
+        position: 'absolute',
+        borderRadius: radius.full,
+        top: '-5px',
+        left: '2px',
+        zIndex: zIndices.hide,
+    }),
+);
+
+export const CheckboxGroupContainer = styled.div<{ isVertical?: boolean }>(({ isVertical }) => ({
+    display: 'flex',
+    flexDirection: isVertical ? 'column' : 'row',
+    justifyContent: 'center',
+    alignItems: 'center',
+    gap: spacing.md,
+    margin: spacing.xxsm,
+}));
diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/index.ts b/datahub-web-react/src/alchemy-components/components/Checkbox/index.ts
new file mode 100644
index 0000000000000..57e3d6d27856a
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Checkbox/index.ts
@@ -0,0 +1,2 @@
+export { Checkbox, CheckboxGroup, checkboxDefaults } from './Checkbox';
+export type { CheckboxProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/types.ts b/datahub-web-react/src/alchemy-components/components/Checkbox/types.ts
new file mode 100644
index 0000000000000..7ee1001168939
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Checkbox/types.ts
@@ -0,0 +1,16 @@
+import { InputHTMLAttributes } from 'react';
+
+export interface CheckboxProps extends InputHTMLAttributes<HTMLInputElement> {
+    label: string;
+    error?: string;
+    isChecked?: boolean;
+    setIsChecked?: React.Dispatch<React.SetStateAction<boolean>>;
+    isDisabled?: boolean;
+    isIntermediate?: boolean;
+    isRequired?: boolean;
+}
+
+export interface CheckboxGroupProps {
+    isVertical?: boolean;
+    checkboxes: CheckboxProps[];
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/utils.ts b/datahub-web-react/src/alchemy-components/components/Checkbox/utils.ts
new file mode 100644
index 0000000000000..edf5d24596e1b
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Checkbox/utils.ts
@@ -0,0 +1,27 @@
+import theme, { colors } from '@components/theme';
+
+const checkboxBackgroundDefault = {
+    default: colors.white,
+    checked: theme.semanticTokens.colors.primary,
+    error: theme.semanticTokens.colors.error,
+    disabled: colors.gray[300],
+};
+
+const checkboxHoverColors = {
+    default: colors.gray[100],
+    error: colors.red[100],
+    checked: colors.violet[100],
+};
+
+export function getCheckboxColor(checked: boolean, error: string, disabled: boolean, mode: 'background' | undefined) {
+    if (disabled) return checkboxBackgroundDefault.disabled;
+    if (error) return checkboxBackgroundDefault.error;
+    if (checked) return checkboxBackgroundDefault.checked;
+    return mode === 'background' ? checkboxBackgroundDefault.default : colors.gray[500];
+}
+
+export function getCheckboxHoverBackgroundColor(checked: boolean, error: string) {
+    if (error) return checkboxHoverColors.error;
+    if (checked) return checkboxHoverColors.checked;
+    return checkboxHoverColors.default;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Heading/Heading.stories.tsx b/datahub-web-react/src/alchemy-components/components/Heading/Heading.stories.tsx
new file mode 100644
index 0000000000000..b8bd9f6420c00
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Heading/Heading.stories.tsx
@@ -0,0 +1,98 @@
+import React from 'react';
+
+import type { Meta, StoryObj, StoryFn } from '@storybook/react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+
+import { VerticalFlexGrid } from '@components/.docs/mdx-components';
+import { Heading, headingDefaults } from '.';
+
+// Auto Docs
+const meta = {
+    title: 'Typography / Heading',
+    component: Heading,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'Used to render semantic HTML heading elements.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        children: {
+            description: 'The content to display within the heading.',
+            table: {
+                type: { summary: 'string' },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        type: {
+            description: 'The type of heading to display.',
+            table: {
+                defaultValue: { summary: headingDefaults.type },
+            },
+        },
+        size: {
+            description: 'Override the size of the heading.',
+            table: {
+                defaultValue: { summary: `${headingDefaults.size}` },
+            },
+        },
+        color: {
+            description: 'Override the color of the heading.',
+            table: {
+                defaultValue: { summary: headingDefaults.color },
+            },
+        },
+        weight: {
+            description: 'Override the weight of the heading.',
+            table: {
+                defaultValue: { summary: `${headingDefaults.weight}` },
+            },
+        },
+    },
+
+    // Define defaults
+    args: {
+        children: 'The content to display within the heading.',
+        type: headingDefaults.type,
+        size: headingDefaults.size,
+        color: headingDefaults.color,
+        weight: headingDefaults.weight,
+    },
+} satisfies Meta<typeof Heading>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+// Basic story is what is displayed 1st in storybook & is used as the code sandbox
+// Pass props to this so that it can be customized via the UI props panel
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Heading {...props}>{props.children}</Heading>,
+};
+
+export const sizes: StoryFn<Story> = (props: any) => (
+    <VerticalFlexGrid>
+        <Heading type="h1">H1 {props.children}</Heading>
+        <Heading type="h2">H2 {props.children}</Heading>
+        <Heading type="h3">H3 {props.children}</Heading>
+        <Heading type="h4">H4 {props.children}</Heading>
+        <Heading type="h5">H5 {props.children}</Heading>
+        <Heading type="h6">H6 {props.children}</Heading>
+    </VerticalFlexGrid>
+);
+
+export const withLink = () => (
+    <Heading>
+        <a href="/">The content to display within the heading</a>
+    </Heading>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Heading/Heading.tsx b/datahub-web-react/src/alchemy-components/components/Heading/Heading.tsx
new file mode 100644
index 0000000000000..6449ff512adac
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Heading/Heading.tsx
@@ -0,0 +1,38 @@
+import React from 'react';
+
+import { HeadingProps } from './types';
+import { H1, H2, H3, H4, H5, H6 } from './components';
+
+export const headingDefaults: HeadingProps = {
+    type: 'h1',
+    color: 'inherit',
+    size: '2xl',
+    weight: 'medium',
+};
+
+export const Heading = ({
+    type = headingDefaults.type,
+    size = headingDefaults.size,
+    color = headingDefaults.color,
+    weight = headingDefaults.weight,
+    children,
+}: HeadingProps) => {
+    const sharedProps = { size, color, weight };
+
+    switch (type) {
+        case 'h1':
+            return <H1 {...sharedProps}>{children}</H1>;
+        case 'h2':
+            return <H2 {...sharedProps}>{children}</H2>;
+        case 'h3':
+            return <H3 {...sharedProps}>{children}</H3>;
+        case 'h4':
+            return <H4 {...sharedProps}>{children}</H4>;
+        case 'h5':
+            return <H5 {...sharedProps}>{children}</H5>;
+        case 'h6':
+            return <H6 {...sharedProps}>{children}</H6>;
+        default:
+            return <H1 {...sharedProps}>{children}</H1>;
+    }
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Heading/components.ts b/datahub-web-react/src/alchemy-components/components/Heading/components.ts
new file mode 100644
index 0000000000000..beea5338585d8
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Heading/components.ts
@@ -0,0 +1,70 @@
+import styled from 'styled-components';
+
+import { typography, colors } from '@components/theme';
+import { getColor, getFontSize } from '@components/theme/utils';
+import { HeadingProps } from './types';
+
+const headingStyles = {
+    H1: {
+        fontSize: typography.fontSizes['4xl'],
+        lineHeight: typography.lineHeights['2xl'],
+    },
+    H2: {
+        fontSize: typography.fontSizes['3xl'],
+        lineHeight: typography.lineHeights.xl,
+    },
+    H3: {
+        fontSize: typography.fontSizes['2xl'],
+        lineHeight: typography.lineHeights.lg,
+    },
+    H4: {
+        fontSize: typography.fontSizes.xl,
+        lineHeight: typography.lineHeights.lg,
+    },
+    H5: {
+        fontSize: typography.fontSizes.lg,
+        lineHeight: typography.lineHeights.md,
+    },
+    H6: {
+        fontSize: typography.fontSizes.md,
+        lineHeight: typography.lineHeights.xs,
+    },
+};
+
+// Default styles
+const baseStyles = {
+    fontFamily: typography.fonts.heading,
+    margin: 0,
+
+    '& a': {
+        color: colors.violet[400],
+        textDecoration: 'none',
+        transition: 'color 0.15s ease',
+
+        '&:hover': {
+            color: colors.violet[500],
+        },
+    },
+};
+
+// Prop Driven Styles
+const propStyles = (props, isText = false) => {
+    const styles = {} as any;
+    if (props.size) styles.fontSize = getFontSize(props.size);
+    if (props.color) styles.color = getColor(props.color);
+    if (props.weight) styles.fontWeight = typography.fontWeights[props.weight];
+    if (isText) styles.lineHeight = typography.lineHeights[props.size];
+    return styles;
+};
+
+// Generate Headings
+const headings = {} as any;
+
+['H1', 'H2', 'H3', 'H4', 'H5', 'H6'].forEach((heading) => {
+    const component = styled[heading.toLowerCase()];
+    headings[heading] = component({ ...baseStyles, ...headingStyles[heading] }, (props: HeadingProps) => ({
+        ...propStyles(props as HeadingProps),
+    }));
+});
+
+export const { H1, H2, H3, H4, H5, H6 } = headings;
diff --git a/datahub-web-react/src/alchemy-components/components/Heading/index.ts b/datahub-web-react/src/alchemy-components/components/Heading/index.ts
new file mode 100644
index 0000000000000..c414de6cc92f7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Heading/index.ts
@@ -0,0 +1,2 @@
+export { Heading, headingDefaults } from './Heading';
+export type { HeadingProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Heading/types.ts b/datahub-web-react/src/alchemy-components/components/Heading/types.ts
new file mode 100644
index 0000000000000..96fcf1ea292bf
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Heading/types.ts
@@ -0,0 +1,9 @@
+import { HTMLAttributes } from 'react';
+import type { FontSizeOptions, FontColorOptions, FontWeightOptions } from '@components/theme/config';
+
+export interface HeadingProps extends HTMLAttributes<HTMLElement> {
+    type?: 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6';
+    size?: FontSizeOptions;
+    color?: FontColorOptions;
+    weight?: FontWeightOptions;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Icon/Icon.stories.tsx b/datahub-web-react/src/alchemy-components/components/Icon/Icon.stories.tsx
new file mode 100644
index 0000000000000..3dcbd74ceb0b7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Icon/Icon.stories.tsx
@@ -0,0 +1,131 @@
+import React from 'react';
+
+import type { Meta, StoryObj } from '@storybook/react';
+
+import { GridList } from '@components/.docs/mdx-components';
+import { Icon, iconDefaults, AVAILABLE_ICONS } from '.';
+
+// Auto Docs
+const meta = {
+    title: 'Media / Icon',
+    component: Icon,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: ['productionReady'],
+        docs: {
+            subtitle: 'A singular component for rendering the icons used throughout the application.',
+            description: {
+                component: '👉 See the [Icons Gallery](/docs/icons--docs) for more information.',
+            },
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        icon: {
+            description: `The name of the icon to display.`,
+            type: 'string',
+            options: AVAILABLE_ICONS,
+            table: {
+                defaultValue: { summary: 'undefined' },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        variant: {
+            description: 'The variant of the icon to display.',
+            defaultValue: 'outline',
+            options: ['outline', 'filled'],
+            table: {
+                defaultValue: { summary: iconDefaults.variant },
+            },
+        },
+        size: {
+            description: 'The size of the icon to display.',
+            defaultValue: 'lg',
+            table: {
+                defaultValue: { summary: iconDefaults.size },
+            },
+        },
+        color: {
+            description: 'The color of the icon to display.',
+            options: ['inherit', 'white', 'black', 'violet', 'green', 'red', 'blue', 'gray'],
+            type: 'string',
+            table: {
+                defaultValue: { summary: iconDefaults.color },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        rotate: {
+            description: 'The rotation of the icon. Applies a CSS transformation.',
+            table: {
+                defaultValue: { summary: iconDefaults.rotate },
+            },
+        },
+    },
+
+    // Define defaults for required args
+    args: {
+        icon: iconDefaults.icon,
+    },
+} satisfies Meta<typeof Icon>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+// Basic story is what is displayed 1st in storybook
+// Pass props to this so that it can be customized via the UI props panel
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Icon {...props} />,
+};
+
+export const filled = () => (
+    <GridList>
+        <Icon icon="AccountCircle" variant="filled" />
+        <Icon icon="AddHome" variant="filled" />
+        <Icon icon="AdminPanelSettings" variant="filled" />
+    </GridList>
+);
+
+export const sizes = () => (
+    <GridList>
+        <Icon icon={iconDefaults.icon} size="xs" />
+        <Icon icon={iconDefaults.icon} size="sm" />
+        <Icon icon={iconDefaults.icon} size="md" />
+        <Icon icon={iconDefaults.icon} size="lg" />
+        <Icon icon={iconDefaults.icon} size="xl" />
+        <Icon icon={iconDefaults.icon} size="2xl" />
+        <Icon icon={iconDefaults.icon} size="3xl" />
+        <Icon icon={iconDefaults.icon} size="4xl" />
+    </GridList>
+);
+
+export const colors = () => (
+    <GridList>
+        <Icon icon={iconDefaults.icon} color="white" />
+        <Icon icon={iconDefaults.icon} color="black" />
+        <Icon icon={iconDefaults.icon} color="violet" />
+        <Icon icon={iconDefaults.icon} color="green" />
+        <Icon icon={iconDefaults.icon} color="red" />
+        <Icon icon={iconDefaults.icon} color="blue" />
+        <Icon icon={iconDefaults.icon} color="gray" />
+    </GridList>
+);
+
+export const rotation = () => (
+    <GridList>
+        <Icon icon="ChevronLeft" />
+        <Icon icon="ChevronLeft" rotate="90" />
+        <Icon icon="ChevronLeft" rotate="180" />
+        <Icon icon="ChevronLeft" rotate="270" />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Icon/Icon.tsx b/datahub-web-react/src/alchemy-components/components/Icon/Icon.tsx
new file mode 100644
index 0000000000000..50c30d7203aed
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Icon/Icon.tsx
@@ -0,0 +1,59 @@
+import React from 'react';
+
+import { getFontSize, getColor, getRotationTransform } from '@components/theme/utils';
+
+import { IconProps } from './types';
+import { IconWrapper } from './components';
+import { getIconNames, getIconComponent } from './utils';
+
+export const iconDefaults: IconProps = {
+    icon: 'AccountCircle',
+    variant: 'outline',
+    size: '4xl',
+    color: 'inherit',
+    rotate: '0',
+};
+
+export const Icon = ({
+    icon,
+    variant = iconDefaults.variant,
+    size = iconDefaults.size,
+    color = iconDefaults.color,
+    rotate = iconDefaults.rotate,
+    ...props
+}: IconProps) => {
+    const { filled, outlined } = getIconNames();
+
+    // Return early if no icon is provided
+    if (!icon) return null;
+
+    // Get outlined icon component name
+    const isOutlined = variant === 'outline';
+    const outlinedIconName = `${icon}Outlined`;
+
+    // Warn if the icon does not have the specified variant
+    if (variant === 'outline' && !outlined.includes(outlinedIconName)) {
+        console.warn(`Icon "${icon}" does not have an outlined variant.`);
+        return null;
+    }
+
+    // Warn if the icon does not have the specified variant
+    if (variant === 'filled' && !filled.includes(icon)) {
+        console.warn(`Icon "${icon}" does not have a filled variant.`);
+        return null;
+    }
+
+    // Get outlined icon component
+    const IconComponent = getIconComponent(isOutlined ? outlinedIconName : icon);
+
+    return (
+        <IconWrapper size={getFontSize(size)} rotate={getRotationTransform(rotate)} {...props}>
+            <IconComponent
+                sx={{
+                    fontSize: getFontSize(size),
+                    color: getColor(color),
+                }}
+            />
+        </IconWrapper>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Icon/components.ts b/datahub-web-react/src/alchemy-components/components/Icon/components.ts
new file mode 100644
index 0000000000000..82e9c9a8fcae0
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Icon/components.ts
@@ -0,0 +1,19 @@
+import styled from 'styled-components';
+
+export const IconWrapper = styled.div<{ size: string; rotate?: string }>`
+    position: relative;
+
+    display: flex;
+    align-items: center;
+    justify-content: center;
+
+    width: ${({ size }) => size};
+    height: ${({ size }) => size};
+
+    & svg {
+        width: 100%;
+        height: 100%;
+
+        transform: ${({ rotate }) => rotate};
+    }
+`;
diff --git a/datahub-web-react/src/alchemy-components/components/Icon/constants.ts b/datahub-web-react/src/alchemy-components/components/Icon/constants.ts
new file mode 100644
index 0000000000000..25145a5970f0f
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Icon/constants.ts
@@ -0,0 +1,547 @@
+export const AVAILABLE_ICONS = [
+    'AccountCircle',
+    'AccountTree',
+    'AddCircle',
+    'AddLink',
+    'Add',
+    'AddTask',
+    'AddToPhotos',
+    'Adjust',
+    'AllInclusive',
+    'Analytics',
+    'Anchor',
+    'Animation',
+    'Announcement',
+    'Api',
+    'Approval',
+    'Archive',
+    'ArrowBack',
+    'ArrowCircleDown',
+    'ArrowCircleLeft',
+    'ArrowCircleRight',
+    'ArrowCircleUp',
+    'ArrowDownward',
+    'ArrowForward',
+    'ArrowOutward',
+    'ArrowUpward',
+    'ArtTrack',
+    'Article',
+    'Assistant',
+    'AttachFile',
+    'Attachment',
+    'AutoAwesome',
+    'AutoFixHigh',
+    'AutoFixOff',
+    'AutoGraph',
+    'AutoMode',
+    'AutoStories',
+    'AvTimer',
+    'Backspace',
+    'Backup',
+    'BackupTable',
+    'Badge',
+    'Balance',
+    'BarChart',
+    'BatchPrediction',
+    'Block',
+    'Bolt',
+    'Book',
+    'BookmarkAdd',
+    'BookmarkAdded',
+    'BookmarkBorder',
+    'Bookmark',
+    'BookmarkRemove',
+    'Bookmarks',
+    'Brush',
+    'BubbleChart',
+    'BugReport',
+    'BuildCircle',
+    'Build',
+    'BusinessCenter',
+    'Business',
+    'Cable',
+    'Cached',
+    'Calculate',
+    'CalendarMonth',
+    'CalendarToday',
+    'CalendarViewDay',
+    'Campaign',
+    'Cancel',
+    'CandlestickChart',
+    'CardGiftcard',
+    'CardMembership',
+    'Cases',
+    'Cast',
+    'Category',
+    'Celebration',
+    'CellTower',
+    'ChangeHistory',
+    'ChatBubble',
+    'Chat',
+    'CheckBox',
+    'CheckCircle',
+    'Check',
+    'Checklist',
+    'ChevronLeft',
+    'ChevronRight',
+    'Class',
+    'CloseFullscreen',
+    'Close',
+    'CloudCircle',
+    'CloudDone',
+    'CloudDownload',
+    'CloudOff',
+    'Cloud',
+    'CloudQueue',
+    'CloudSync',
+    'CloudUpload',
+    'CoPresent',
+    'CodeOff',
+    'Code',
+    'ColorLens',
+    'Colorize',
+    'CommentBank',
+    'Comment',
+    'CommentsDisabled',
+    'Commit',
+    'CompareArrows',
+    'Compare',
+    'Compress',
+    'Computer',
+    'Construction',
+    'ContactPage',
+    'ContactSupport',
+    'Contacts',
+    'ContentCopy',
+    'ContentCut',
+    'Contrast',
+    'ControlPoint',
+    'Cookie',
+    'CopyAll',
+    'Copyright',
+    'CorporateFare',
+    'Cottage',
+    'CreateNewFolder',
+    'CrisisAlert',
+    'Cyclone',
+    'Dangerous',
+    'DarkMode',
+    'DashboardCustomize',
+    'Dashboard',
+    'DataArray',
+    'DataObject',
+    'DataThresholding',
+    'DataUsage',
+    'DatasetLinked',
+    'Dataset',
+    'DateRange',
+    'DeleteForever',
+    'Delete',
+    'DeleteSweep',
+    'Description',
+    'Deselect',
+    'DesignServices',
+    'Details',
+    'DeviceHub',
+    'DeviceThermostat',
+    'Diamond',
+    'Difference',
+    'DisabledByDefault',
+    'DiscFull',
+    'Discount',
+    'DisplaySettings',
+    'Diversity2',
+    'Dns',
+    'DoNotDisturb',
+    'DocumentScanner',
+    'DomainAdd',
+    'DomainDisabled',
+    'Domain',
+    'DomainVerification',
+    'DoneAll',
+    'DonutLarge',
+    'DonutSmall',
+    'DoubleArrow',
+    'DownloadDone',
+    'DownloadForOffline',
+    'Download',
+    'Downloading',
+    'Drafts',
+    'DragHandle',
+    'DragIndicator',
+    'Draw',
+    'DriveFileMove',
+    'DriveFolderUpload',
+    'DynamicFeed',
+    'DynamicForm',
+    'EditCalendar',
+    'EditLocation',
+    'EditNote',
+    'EditOff',
+    'Edit',
+    'Eject',
+    'ElectricBolt',
+    'EmergencyShare',
+    'EnhancedEncryption',
+    'Equalizer',
+    'Error',
+    'EventAvailable',
+    'EventBusy',
+    'EventNote',
+    'Event',
+    'EventRepeat',
+    'ExitToApp',
+    'Expand',
+    'ExploreOff',
+    'Explore',
+    'Exposure',
+    'ExtensionOff',
+    'Extension',
+    'FastForward',
+    'FastRewind',
+    'FavoriteBorder',
+    'Favorite',
+    'FeaturedPlayList',
+    'Feed',
+    'Feedback',
+    'FileCopy',
+    'FileDownloadOff',
+    'FileDownload',
+    'FileOpen',
+    'FilePresent',
+    'FileUpload',
+    'FilterAltOff',
+    'FilterAlt',
+    'FilterListOff',
+    'FindInPage',
+    'FindReplace',
+    'FirstPage',
+    'FitScreen',
+    'FlagCircle',
+    'Flag',
+    'Flaky',
+    'Flare',
+    'FlashOff',
+    'FlashOn',
+    'FlightLand',
+    'Flight',
+    'FlightTakeoff',
+    'FmdBad',
+    'FmdGood',
+    'FolderCopy',
+    'FolderDelete',
+    'FolderOff',
+    'FolderOpen',
+    'Folder',
+    'FolderShared',
+    'FolderSpecial',
+    'FolderZip',
+    'ForkLeft',
+    'ForkRight',
+    'FormatListBulleted',
+    'FormatListNumbered',
+    'Forum',
+    'FullscreenExit',
+    'Fullscreen',
+    'Functions',
+    'GetApp',
+    'GppBad',
+    'GppGood',
+    'GppMaybe',
+    'GpsFixed',
+    'GpsNotFixed',
+    'GpsOff',
+    'Grading',
+    'Grain',
+    'GraphicEq',
+    'Grid3x3',
+    'Grid4x4',
+    'GridGoldenratio',
+    'GridOff',
+    'GridOn',
+    'GridView',
+    'GroupAdd',
+    'Group',
+    'GroupRemove',
+    'GroupWork',
+    'Groups',
+    'Handshake',
+    'Handyman',
+    'Hardware',
+    'HealthAndSafety',
+    'HelpCenter',
+    'Help',
+    'Hexagon',
+    'HideSource',
+    'Highlight',
+    'History',
+    'HistoryToggleOff',
+    'Home',
+    'Hub',
+    'Image',
+    'ImageSearch',
+    'Inbox',
+    'Info',
+    'Input',
+    'InsertChart',
+    'InsertComment',
+    'InsertDriveFile',
+    'Insights',
+    'Interests',
+    'Inventory2',
+    'Inventory',
+    'KeyOff',
+    'Key',
+    'LabelImportant',
+    'LabelOff',
+    'Label',
+    'Lan',
+    'Landscape',
+    'Language',
+    'LastPage',
+    'Launch',
+    'LayersClear',
+    'Layers',
+    'Leaderboard',
+    'LegendToggle',
+    'LibraryAddCheck',
+    'LibraryAdd',
+    'LightMode',
+    'Lightbulb',
+    'LineAxis',
+    'LineStyle',
+    'LineWeight',
+    'LinearScale',
+    'LinkOff',
+    'Link',
+    'List',
+    'LockOpen',
+    'Lock',
+    'LockReset',
+    'Login',
+    'Logout',
+    'Loupe',
+    'LowPriority',
+    'Loyalty',
+    'Mail',
+    'ManageAccounts',
+    'ManageHistory',
+    'ManageSearch',
+    'Map',
+    'MapsUgc',
+    'MarkAsUnread',
+    'MeetingRoom',
+    'Memory',
+    'MenuBook',
+    'MenuOpen',
+    'Menu',
+    'Merge',
+    'MergeType',
+    'Message',
+    'MiscellaneousServices',
+    'MoodBad',
+    'Mood',
+    'MoreHoriz',
+    'MoreTime',
+    'MoreVert',
+    'MoveDown',
+    'MoveToInbox',
+    'MoveUp',
+    'MultilineChart',
+    'MultipleStop',
+    'Nat',
+    'NewReleases',
+    'NightsStay',
+    'NoAccounts',
+    'NoEncryption',
+    'NotStarted',
+    'NoteAdd',
+    'NotificationAdd',
+    'NotificationImportant',
+    'NotificationsActive',
+    'NotificationsOff',
+    'Notifications',
+    'NotificationsPaused',
+    'OpenInFull',
+    'OpenInNew',
+    'Outbound',
+    'Outbox',
+    'Output',
+    'Pageview',
+    'Password',
+    'PauseCircle',
+    'PendingActions',
+    'Pending',
+    'People',
+    'PersonAddAlt1',
+    'PersonOff',
+    'Person',
+    'PersonRemoveAlt1',
+    'PersonSearch',
+    'PinDrop',
+    'PivotTableChart',
+    'Place',
+    'PlayArrow',
+    'PlayCircle',
+    'Policy',
+    'Poll',
+    'Polyline',
+    'PostAdd',
+    'Preview',
+    'PrivacyTip',
+    'PublicOff',
+    'Public',
+    'Publish',
+    'PushPin',
+    'QueryStats',
+    'QuestionAnswer',
+    'Queue',
+    'Radar',
+    'ReadMore',
+    'Redo',
+    'Refresh',
+    'RemoveCircle',
+    'Replay',
+    'ReplyAll',
+    'Reply',
+    'Report',
+    'ReportProblem',
+    'Restore',
+    'RocketLaunch',
+    'Rocket',
+    'Route',
+    'RssFeed',
+    'Rule',
+    'RunningWithErrors',
+    'SatelliteAlt',
+    'SaveAlt',
+    'Schedule',
+    'Schema',
+    'Science',
+    'SearchOff',
+    'Search',
+    'Security',
+    'Sell',
+    'Sensors',
+    'SentimentDissatisfied',
+    'SentimentNeutral',
+    'SentimentSatisfied',
+    'Settings',
+    'Share',
+    'Shield',
+    'ShortText',
+    'Shortcut',
+    'ShowChart',
+    'Shuffle',
+    'Signpost',
+    'SkipNext',
+    'SkipPrevious',
+    'SortByAlpha',
+    'Sort',
+    'Source',
+    'SpaceDashboard',
+    'Speed',
+    'SsidChart',
+    'StackedBarChart',
+    'StackedLineChart',
+    'StarBorder',
+    'StarHalf',
+    'Star',
+    'Start',
+    'StickyNote2',
+    'StopCircle',
+    'Storage',
+    'Storm',
+    'Straight',
+    'Stream',
+    'Style',
+    'SubdirectoryArrowLeft',
+    'SubdirectoryArrowRight',
+    'Subject',
+    'Subscriptions',
+    'SubtitlesOff',
+    'Support',
+    'SwapHoriz',
+    'SwapVert',
+    'SwitchAccount',
+    'SwitchLeft',
+    'SwitchRight',
+    'SyncAlt',
+    'SyncDisabled',
+    'SyncLock',
+    'Sync',
+    'SyncProblem',
+    'TableChart',
+    'TableRows',
+    'TableView',
+    'Tag',
+    'TaskAlt',
+    'Terminal',
+    'ThumbDown',
+    'ThumbUp',
+    'ThumbsUpDown',
+    'Timelapse',
+    'Timeline',
+    'TipsAndUpdates',
+    'Toc',
+    'TrackChanges',
+    'TrendingDown',
+    'TrendingFlat',
+    'TrendingUp',
+    'Tune',
+    'Tungsten',
+    'TurnLeft',
+    'TurnRight',
+    'TurnSlightLeft',
+    'TurnSlightRight',
+    'Unarchive',
+    'Undo',
+    'UnfoldLessDouble',
+    'UnfoldLess',
+    'UnfoldMoreDouble',
+    'UnfoldMore',
+    'Unsubscribe',
+    'Upcoming',
+    'UpdateDisabled',
+    'Update',
+    'Upgrade',
+    'UploadFile',
+    'Upload',
+    'Verified',
+    'VerifiedUser',
+    'ViewAgenda',
+    'ViewArray',
+    'ViewCarousel',
+    'ViewColumn',
+    'ViewComfy',
+    'ViewCompact',
+    'ViewCozy',
+    'ViewDay',
+    'ViewHeadline',
+    'ViewKanban',
+    'ViewList',
+    'ViewModule',
+    'ViewQuilt',
+    'ViewSidebar',
+    'ViewStream',
+    'ViewTimeline',
+    'ViewWeek',
+    'VisibilityOff',
+    'Visibility',
+    'Warehouse',
+    'Warning',
+    'Webhook',
+    'Whatshot',
+    'Widgets',
+    'Wifi',
+    'Window',
+    'WorkHistory',
+    'WorkOff',
+    'Work',
+    'WorkspacePremium',
+    'Workspaces',
+    'Wysiwyg',
+    'ZoomInMap',
+    'ZoomIn',
+    'ZoomOutMap',
+];
diff --git a/datahub-web-react/src/alchemy-components/components/Icon/index.ts b/datahub-web-react/src/alchemy-components/components/Icon/index.ts
new file mode 100644
index 0000000000000..23ca0a7ef7da2
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Icon/index.ts
@@ -0,0 +1,3 @@
+export { Icon, iconDefaults } from './Icon';
+export type { IconProps, IconNames } from './types';
+export { AVAILABLE_ICONS } from './constants';
diff --git a/datahub-web-react/src/alchemy-components/components/Icon/types.ts b/datahub-web-react/src/alchemy-components/components/Icon/types.ts
new file mode 100644
index 0000000000000..f5a050e9338a7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Icon/types.ts
@@ -0,0 +1,23 @@
+import { HTMLAttributes } from 'react';
+
+import type { FontSizeOptions, FontColorOptions, RotationOptions } from '@components/theme/config';
+import { AVAILABLE_ICONS } from './constants';
+
+// Utility function to create an enum from an array of strings
+function createEnum<T extends string>(values: T[]): { [K in T]: K } {
+    return values.reduce((acc, value) => {
+        acc[value] = value;
+        return acc;
+    }, Object.create(null));
+}
+
+const names = createEnum(AVAILABLE_ICONS);
+export type IconNames = keyof typeof names;
+
+export interface IconProps extends HTMLAttributes<HTMLElement> {
+    icon: IconNames;
+    variant?: 'filled' | 'outline';
+    size?: FontSizeOptions;
+    color?: FontColorOptions;
+    rotate?: RotationOptions;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Icon/utils.ts b/datahub-web-react/src/alchemy-components/components/Icon/utils.ts
new file mode 100644
index 0000000000000..1137b3da28bc7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Icon/utils.ts
@@ -0,0 +1,29 @@
+import * as materialIcons from '@mui/icons-material';
+
+export const getIconNames = () => {
+    // We only want "Filled" (mui default) and "Outlined" icons
+    const filtered = Object.keys(materialIcons).filter(
+        (key) =>
+            !key.includes('Filled') && !key.includes('TwoTone') && !key.includes('Rounded') && !key.includes('Sharp'),
+    );
+
+    const filled: string[] = [];
+    const outlined: string[] = [];
+
+    filtered.forEach((key) => {
+        if (key.includes('Outlined')) {
+            outlined.push(key);
+        } else if (!key.includes('Outlined')) {
+            filled.push(key);
+        }
+    });
+
+    return {
+        filled,
+        outlined,
+    };
+};
+
+export const getIconComponent = (icon: string) => {
+    return materialIcons[icon];
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Input/Input.stories.tsx b/datahub-web-react/src/alchemy-components/components/Input/Input.stories.tsx
new file mode 100644
index 0000000000000..053e952b62a2e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Input/Input.stories.tsx
@@ -0,0 +1,177 @@
+import React from 'react';
+
+import type { Meta, StoryObj } from '@storybook/react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+
+import { GridList } from '@components/.docs/mdx-components';
+import { AVAILABLE_ICONS } from '../Icon';
+
+import { Input, inputDefaults } from './Input';
+
+const meta = {
+    title: 'Forms / Input',
+    component: Input,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'A component that is used to get user input in a single line field.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        value: {
+            description: 'Value for the Input.',
+            table: {
+                defaultValue: { summary: inputDefaults.value as string },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        label: {
+            description: 'Label for the Input.',
+            table: {
+                defaultValue: { summary: inputDefaults.label },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        placeholder: {
+            description: 'Placeholder for the Input.',
+            table: {
+                defaultValue: { summary: inputDefaults.placeholder },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        icon: {
+            description: 'The icon to display in the Input.',
+            type: 'string',
+            options: AVAILABLE_ICONS,
+            table: {
+                defaultValue: { summary: 'undefined' },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        error: {
+            description: 'Enforce error state on the Input.',
+            table: {
+                defaultValue: { summary: inputDefaults.error },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        warning: {
+            description: 'Enforce warning state on the Input.',
+            table: {
+                defaultValue: { summary: inputDefaults.warning },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        isSuccess: {
+            description: 'Enforce success state on the Input.',
+            table: {
+                defaultValue: { summary: inputDefaults?.isSuccess?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isDisabled: {
+            description: 'Whether the Input is in disabled state.',
+            table: {
+                defaultValue: { summary: inputDefaults?.isDisabled?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isInvalid: {
+            description: 'Whether the Input is an invalid state.',
+            table: {
+                defaultValue: { summary: inputDefaults?.isInvalid?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isReadOnly: {
+            description: 'Whether the Input is in readonly mode.',
+            table: {
+                defaultValue: { summary: inputDefaults?.isReadOnly?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isPassword: {
+            description: 'Whether the Input has a password type.',
+            table: {
+                defaultValue: { summary: inputDefaults?.isPassword?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isRequired: {
+            description: 'Whether the Input is a required field.',
+            table: {
+                defaultValue: { summary: inputDefaults?.isRequired?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+    },
+    args: {
+        value: inputDefaults.value,
+        label: inputDefaults.label,
+        placeholder: inputDefaults.placeholder,
+        icon: inputDefaults.icon,
+        error: inputDefaults.error,
+        warning: inputDefaults.warning,
+        isSuccess: inputDefaults.isSuccess,
+        isDisabled: inputDefaults.isDisabled,
+        isInvalid: inputDefaults.isInvalid,
+        isReadOnly: inputDefaults.isReadOnly,
+        isPassword: inputDefaults.isPassword,
+        isRequired: inputDefaults.isRequired,
+    },
+} satisfies Meta<typeof Input>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Input {...props} />,
+};
+
+export const status = () => (
+    <GridList width="400px" isVertical>
+        <Input label="Success Status" isSuccess />
+        <Input label="Error Status" isInvalid error="Error Message" />
+        <Input label="Warning Status" warning="Warning Message" />
+    </GridList>
+);
+
+export const states = () => (
+    <GridList width="400px" isVertical>
+        <Input label="Disabled" isDisabled />
+        <Input label="Readonly" isReadOnly />
+        <Input label="Required" isRequired />
+        <Input label="Password" isPassword />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Input/Input.tsx b/datahub-web-react/src/alchemy-components/components/Input/Input.tsx
new file mode 100644
index 0000000000000..976fc47ffc594
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Input/Input.tsx
@@ -0,0 +1,97 @@
+import { Tooltip } from '@components';
+import React from 'react';
+
+import { InputProps } from './types';
+
+import { ErrorMessage, InputContainer, InputField, InputWrapper, Label, Required, WarningMessage } from './components';
+
+import { Icon } from '../Icon';
+import { getInputType } from './utils';
+
+export const inputDefaults: InputProps = {
+    value: '',
+    setValue: () => {},
+    label: 'Label',
+    placeholder: 'Placeholder',
+    error: '',
+    warning: '',
+    isSuccess: false,
+    isDisabled: false,
+    isInvalid: false,
+    isReadOnly: false,
+    isPassword: false,
+    isRequired: false,
+    errorOnHover: false,
+    type: 'text',
+};
+
+export const Input = ({
+    value = inputDefaults.value,
+    setValue = inputDefaults.setValue,
+    label = inputDefaults.label,
+    placeholder = inputDefaults.placeholder,
+    icon, // default undefined
+    error = inputDefaults.error,
+    warning = inputDefaults.warning,
+    isSuccess = inputDefaults.isSuccess,
+    isDisabled = inputDefaults.isDisabled,
+    isInvalid = inputDefaults.isInvalid,
+    isReadOnly = inputDefaults.isReadOnly,
+    isPassword = inputDefaults.isPassword,
+    isRequired = inputDefaults.isRequired,
+    errorOnHover = inputDefaults.errorOnHover,
+    type = inputDefaults.type,
+    id,
+    ...props
+}: InputProps) => {
+    // Invalid state is always true if error is present
+    let invalid = isInvalid;
+    if (error) invalid = true;
+
+    // Show/hide password text
+    const [showPassword, setShowPassword] = React.useState(false);
+    const passwordIcon = showPassword ? 'Visibility' : 'VisibilityOff';
+
+    // Input base props
+    const inputBaseProps = {
+        label,
+        isSuccess,
+        error,
+        warning,
+        isDisabled,
+        isInvalid: invalid,
+    };
+
+    return (
+        <InputWrapper {...props}>
+            {label && (
+                <Label aria-label={label}>
+                    {label} {isRequired && <Required>*</Required>}
+                </Label>
+            )}
+            <InputContainer {...inputBaseProps}>
+                {icon && <Icon icon={icon} size="lg" />}
+                <InputField
+                    value={value}
+                    onChange={(e) => setValue?.(e.target.value)}
+                    type={getInputType(type, isPassword, showPassword)}
+                    placeholder={placeholder}
+                    readOnly={isReadOnly}
+                    disabled={isDisabled}
+                    required={isRequired}
+                    id={id}
+                />
+                {!isPassword && (
+                    <Tooltip title={errorOnHover ? error : ''} showArrow={false}>
+                        {invalid && <Icon icon="WarningAmber" color="red" size="lg" />}
+                        {isSuccess && <Icon icon="CheckCircle" color="green" size="lg" />}
+                        {warning && <Icon icon="ErrorOutline" color="yellow" size="lg" />}
+                    </Tooltip>
+                )}
+                {isPassword && <Icon onClick={() => setShowPassword(!showPassword)} icon={passwordIcon} size="lg" />}
+            </InputContainer>
+            {invalid && error && !errorOnHover && <ErrorMessage>{error}</ErrorMessage>}
+            {warning && <WarningMessage>{warning}</WarningMessage>}
+        </InputWrapper>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Input/components.ts b/datahub-web-react/src/alchemy-components/components/Input/components.ts
new file mode 100644
index 0000000000000..d1c337642d9cd
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Input/components.ts
@@ -0,0 +1,92 @@
+import styled from 'styled-components';
+
+import theme, { borders, colors, radius, spacing, typography } from '@components/theme';
+import { getStatusColors } from '@components/theme/utils';
+
+import {
+    INPUT_MAX_HEIGHT,
+    formLabelTextStyles,
+    inputValueTextStyles,
+    inputPlaceholderTextStyles,
+} from '../commonStyles';
+
+import type { InputProps } from './types';
+
+const defaultFlexStyles = {
+    display: 'flex',
+    justifyContent: 'space-between',
+    alignItems: 'center',
+};
+
+const defaultMessageStyles = {
+    marginTop: spacing.xxsm,
+    fontSize: typography.fontSizes.sm,
+};
+
+export const InputWrapper = styled.div({
+    ...defaultFlexStyles,
+    alignItems: 'flex-start',
+    flexDirection: 'column',
+    width: '100%',
+});
+
+export const InputContainer = styled.div(
+    ({ isSuccess, warning, isDisabled, isInvalid }: InputProps) => ({
+        border: `${borders['1px']} ${getStatusColors(isSuccess, warning, isInvalid)}`,
+        backgroundColor: isDisabled ? colors.gray[100] : colors.white,
+        paddingRight: spacing.md,
+    }),
+    {
+        ...defaultFlexStyles,
+        width: '100%',
+        maxHeight: INPUT_MAX_HEIGHT,
+        overflow: 'hidden',
+        borderRadius: radius.md,
+        flex: 1,
+        color: colors.gray[400], // 1st icon color
+
+        '&:focus-within': {
+            borderColor: colors.violet[200],
+            outline: `${borders['1px']} ${colors.violet[200]}`,
+        },
+    },
+);
+
+export const InputField = styled.input({
+    padding: `${spacing.sm} ${spacing.md}`,
+    lineHeight: typography.lineHeights.normal,
+    maxHeight: INPUT_MAX_HEIGHT,
+    border: borders.none,
+    width: '100%',
+
+    // Shared common input text styles
+    ...inputValueTextStyles(),
+
+    '&::placeholder': {
+        ...inputPlaceholderTextStyles,
+    },
+
+    '&:focus': {
+        outline: 'none',
+    },
+});
+
+export const Required = styled.span({
+    color: colors.red[500],
+});
+
+export const Label = styled.div({
+    ...formLabelTextStyles,
+    marginBottom: spacing.xsm,
+    textAlign: 'left',
+});
+
+export const ErrorMessage = styled.div({
+    ...defaultMessageStyles,
+    color: theme.semanticTokens.colors.error,
+});
+
+export const WarningMessage = styled.div({
+    ...defaultMessageStyles,
+    color: theme.semanticTokens.colors.warning,
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Input/index.ts b/datahub-web-react/src/alchemy-components/components/Input/index.ts
new file mode 100644
index 0000000000000..336a9b4dd08e9
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Input/index.ts
@@ -0,0 +1,2 @@
+export { Input, inputDefaults } from './Input';
+export type { InputProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Input/types.ts b/datahub-web-react/src/alchemy-components/components/Input/types.ts
new file mode 100644
index 0000000000000..1b2abf132d328
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Input/types.ts
@@ -0,0 +1,22 @@
+import { InputHTMLAttributes } from 'react';
+
+import { IconNames } from '../Icon';
+
+export interface InputProps extends InputHTMLAttributes<HTMLInputElement> {
+    value?: string | number | readonly string[] | undefined;
+    setValue?: React.Dispatch<React.SetStateAction<string>>;
+    label: string;
+    placeholder?: string;
+    icon?: IconNames;
+    error?: string;
+    warning?: string;
+    isSuccess?: boolean;
+    isDisabled?: boolean;
+    isInvalid?: boolean;
+    isReadOnly?: boolean;
+    isPassword?: boolean;
+    isRequired?: boolean;
+    errorOnHover?: boolean;
+    id?: string;
+    type?: string;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Input/utils.ts b/datahub-web-react/src/alchemy-components/components/Input/utils.ts
new file mode 100644
index 0000000000000..142a93232485b
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Input/utils.ts
@@ -0,0 +1,5 @@
+export const getInputType = (type?: string, isPassword?: boolean, showPassword?: boolean) => {
+    if (type) return type;
+    if (isPassword && !showPassword) return 'password';
+    return 'text';
+};
diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.stories.tsx b/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.stories.tsx
new file mode 100644
index 0000000000000..8cce0369918a2
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.stories.tsx
@@ -0,0 +1,96 @@
+import React from 'react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import type { Meta, StoryObj } from '@storybook/react';
+import { LineChart } from './LineChart';
+import { getMockedProps } from '../BarChart/utils';
+
+const meta = {
+    title: 'Charts / LineChart',
+    component: LineChart,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.EXPERIMENTAL],
+        docs: {
+            subtitle: 'A component that is used to show LineChart',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        data: {
+            description: 'Array of datum to show',
+        },
+        xAccessor: {
+            description: 'A function to convert datum to value of X',
+        },
+        yAccessor: {
+            description: 'A function to convert datum to value of Y',
+        },
+        renderTooltipContent: {
+            description: 'A function to replace default rendering of toolbar',
+        },
+        margin: {
+            description: 'Add margins to chart',
+        },
+        leftAxisTickFormat: {
+            description: 'A function to format labels of left axis',
+        },
+        leftAxisTickLabelProps: {
+            description: 'Props for label of left axis',
+        },
+        bottomAxisTickFormat: {
+            description: 'A function to format labels of bottom axis',
+        },
+        bottomAxisTickLabelProps: {
+            description: 'Props for label of bottom axis',
+        },
+        lineColor: {
+            description: 'Color of line on chart',
+            control: {
+                type: 'color',
+            },
+        },
+        areaColor: {
+            description: 'Color of area under line',
+            control: {
+                type: 'color',
+            },
+        },
+        gridColor: {
+            description: "Color of grid's lines",
+            control: {
+                type: 'color',
+            },
+        },
+        renderGradients: {
+            description: 'A function to render different gradients that can be used as colors',
+        },
+        toolbarVerticalCrosshairStyle: {
+            description: "Styles of toolbar's vertical line",
+        },
+        renderTooltipGlyph: {
+            description: 'A function to render a glyph',
+        },
+    },
+
+    // Define defaults
+    args: {
+        ...getMockedProps(),
+        renderTooltipContent: (datum) => <>DATUM: {JSON.stringify(datum)}</>,
+    },
+} satisfies Meta<typeof LineChart>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => (
+        <div style={{ width: '900px', height: '350px' }}>
+            <LineChart {...props} />
+        </div>
+    ),
+};
diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.tsx b/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.tsx
new file mode 100644
index 0000000000000..22580122ccf84
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.tsx
@@ -0,0 +1,178 @@
+import { colors } from '@src/alchemy-components/theme';
+// import { abbreviateNumber } from '@src/app/dataviz/utils';
+import { TickLabelProps } from '@visx/axis';
+import { curveMonotoneX } from '@visx/curve';
+import { LinearGradient } from '@visx/gradient';
+import { ParentSize } from '@visx/responsive';
+import { AreaSeries, Axis, AxisScale, Grid, LineSeries, Tooltip, XYChart } from '@visx/xychart';
+import dayjs from 'dayjs';
+import React, { useState } from 'react';
+import { Popover } from '../Popover';
+import { ChartWrapper } from './components';
+import { LineChartProps } from './types';
+import { abbreviateNumber } from '../dataviz/utils';
+
+const commonTickLabelProps: TickLabelProps<any> = {
+    fontSize: 10,
+    fontFamily: 'Mulish',
+    fill: colors.gray[1700],
+};
+
+const GLYPH_DROP_SHADOW_FILTER = `
+    drop-shadow(0px 1px 3px rgba(33, 23, 95, 0.30))
+    drop-shadow(0px 2px 5px rgba(33, 23, 95, 0.25))
+    drop-shadow(0px -2px 5px rgba(33, 23, 95, 0.25)
+`;
+
+export const lineChartDefault: LineChartProps<any> = {
+    data: [],
+    xAccessor: (datum) => datum?.x,
+    yAccessor: (datum) => datum?.y,
+    leftAxisTickFormat: abbreviateNumber,
+    leftAxisTickLabelProps: {
+        ...commonTickLabelProps,
+        textAnchor: 'end',
+    },
+    bottomAxisTickFormat: (x) => dayjs(x).format('D MMM'),
+    bottomAxisTickLabelProps: {
+        ...commonTickLabelProps,
+        textAnchor: 'middle',
+        verticalAnchor: 'start',
+    },
+    lineColor: colors.violet[500],
+    areaColor: 'url(#line-gradient)',
+    gridColor: '#e0e0e0',
+    renderGradients: () => (
+        <LinearGradient id="line-gradient" from={colors.violet[200]} to={colors.white} toOpacity={0.6} />
+    ),
+    toolbarVerticalCrosshairStyle: {
+        stroke: colors.white,
+        strokeWidth: 2,
+        filter: GLYPH_DROP_SHADOW_FILTER,
+    },
+    renderTooltipGlyph: (props) => {
+        return (
+            <>
+                <circle cx={props.x} cy={props.y} r="8" fill={colors.white} filter={GLYPH_DROP_SHADOW_FILTER} />
+                <circle cx={props.x} cy={props.y} r="6" fill={colors.violet[500]} />
+            </>
+        );
+    },
+};
+
+export function LineChart<DatumType extends object>({
+    data,
+    xAccessor = lineChartDefault.xAccessor,
+    yAccessor = lineChartDefault.yAccessor,
+    renderTooltipContent,
+    margin,
+    leftAxisTickFormat = lineChartDefault.leftAxisTickFormat,
+    leftAxisTickLabelProps = lineChartDefault.leftAxisTickLabelProps,
+    bottomAxisTickFormat = lineChartDefault.bottomAxisTickFormat,
+    bottomAxisTickLabelProps = lineChartDefault.bottomAxisTickLabelProps,
+    lineColor = lineChartDefault.lineColor,
+    areaColor = lineChartDefault.areaColor,
+    gridColor = lineChartDefault.gridColor,
+    renderGradients = lineChartDefault.renderGradients,
+    toolbarVerticalCrosshairStyle = lineChartDefault.toolbarVerticalCrosshairStyle,
+    renderTooltipGlyph = lineChartDefault.renderTooltipGlyph,
+}: LineChartProps<DatumType>) {
+    const [showGrid, setShowGrid] = useState<boolean>(false);
+
+    // FYI: additional margins to show left and bottom axises
+    const internalMargin = {
+        top: (margin?.top ?? 0) + 30,
+        right: (margin?.right ?? 0) + 20,
+        bottom: (margin?.bottom ?? 0) + 35,
+        left: (margin?.left ?? 0) + 40,
+    };
+
+    const accessors = { xAccessor, yAccessor };
+
+    return (
+        <ChartWrapper onMouseEnter={() => setShowGrid(true)} onMouseLeave={() => setShowGrid(false)}>
+            <ParentSize>
+                {({ width, height }) => {
+                    return (
+                        <XYChart
+                            width={width}
+                            height={height}
+                            xScale={{ type: 'time' }}
+                            yScale={{ type: 'linear', nice: true, round: true }}
+                            margin={internalMargin}
+                            captureEvents
+                        >
+                            {renderGradients?.()}
+
+                            <Axis
+                                orientation="left"
+                                tickFormat={leftAxisTickFormat}
+                                tickLabelProps={leftAxisTickLabelProps}
+                                hideAxisLine
+                                hideTicks
+                            />
+
+                            <Axis
+                                orientation="bottom"
+                                numTicks={Math.floor(data.length / 2)}
+                                tickFormat={bottomAxisTickFormat}
+                                tickLabelProps={bottomAxisTickLabelProps}
+                                hideAxisLine
+                                hideTicks
+                            />
+
+                            <line
+                                x1={internalMargin.left}
+                                x2={internalMargin.left}
+                                y1={0}
+                                y2={height - internalMargin.bottom}
+                                stroke={gridColor}
+                            />
+
+                            {showGrid && (
+                                <Grid rows={false} columns stroke={gridColor} numTicks={data.length} lineStyle={{}} />
+                            )}
+
+                            <AreaSeries<AxisScale, AxisScale, DatumType>
+                                dataKey="line-chart-seria-01"
+                                data={data}
+                                fill={areaColor}
+                                curve={curveMonotoneX}
+                                {...accessors}
+                            />
+                            <LineSeries<AxisScale, AxisScale, DatumType>
+                                dataKey="line-chart-seria-01"
+                                data={data}
+                                stroke={lineColor}
+                                curve={curveMonotoneX}
+                                {...accessors}
+                            />
+
+                            <Tooltip<DatumType>
+                                snapTooltipToDatumX
+                                snapTooltipToDatumY
+                                showVerticalCrosshair
+                                applyPositionStyle
+                                showSeriesGlyphs
+                                verticalCrosshairStyle={toolbarVerticalCrosshairStyle}
+                                renderGlyph={renderTooltipGlyph}
+                                unstyled
+                                renderTooltip={({ tooltipData }) => {
+                                    return (
+                                        tooltipData?.nearestDatum && (
+                                            <Popover
+                                                open
+                                                placement="topLeft"
+                                                content={renderTooltipContent?.(tooltipData.nearestDatum.datum)}
+                                            />
+                                        )
+                                    );
+                                }}
+                            />
+                        </XYChart>
+                    );
+                }}
+            </ParentSize>
+        </ChartWrapper>
+    );
+}
diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/components.tsx b/datahub-web-react/src/alchemy-components/components/LineChart/components.tsx
new file mode 100644
index 0000000000000..fb6c0cf1ced78
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LineChart/components.tsx
@@ -0,0 +1,8 @@
+import styled from 'styled-components';
+
+export const ChartWrapper = styled.div`
+    width: 100%;
+    height: 100%;
+    position: relative;
+    cursor: pointer;
+`;
diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/index.ts b/datahub-web-react/src/alchemy-components/components/LineChart/index.ts
new file mode 100644
index 0000000000000..7fca9300d578c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LineChart/index.ts
@@ -0,0 +1 @@
+export { LineChart } from './LineChart';
diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/types.ts b/datahub-web-react/src/alchemy-components/components/LineChart/types.ts
new file mode 100644
index 0000000000000..cf45662ba7cf9
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LineChart/types.ts
@@ -0,0 +1,22 @@
+import { TickFormatter, TickLabelProps } from '@visx/axis';
+import { Margin } from '@visx/xychart';
+import { RenderTooltipGlyphProps } from '@visx/xychart/lib/components/Tooltip';
+import React from 'react';
+
+export type LineChartProps<DatumType extends object> = {
+    data: DatumType[];
+    xAccessor: (datum: DatumType) => string | number;
+    yAccessor: (datum: DatumType) => number;
+    renderTooltipContent?: (datum: DatumType) => React.ReactNode;
+    margin?: Margin;
+    leftAxisTickFormat?: TickFormatter<DatumType>;
+    leftAxisTickLabelProps?: TickLabelProps<DatumType>;
+    bottomAxisTickFormat?: TickFormatter<DatumType>;
+    bottomAxisTickLabelProps?: TickLabelProps<DatumType>;
+    lineColor?: string;
+    areaColor?: string;
+    gridColor?: string;
+    renderGradients?: () => React.ReactNode;
+    toolbarVerticalCrosshairStyle?: React.SVGProps<SVGLineElement>;
+    renderTooltipGlyph?: (props: RenderTooltipGlyphProps<object>) => React.ReactNode | undefined;
+};
diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.stories.tsx b/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.stories.tsx
new file mode 100644
index 0000000000000..7016ecbc7c90a
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.stories.tsx
@@ -0,0 +1,71 @@
+import React from 'react';
+
+import type { Meta, StoryObj } from '@storybook/react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import { PageTitle } from '.';
+
+// Auto Docs
+const meta = {
+    title: 'Pages / Page Title',
+    component: PageTitle,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'Used to render the title and subtitle for a page.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        title: {
+            description: 'The title text',
+        },
+        subTitle: {
+            description: 'The subtitle text',
+        },
+        variant: {
+            description: 'The variant of header based on its usage',
+        },
+    },
+
+    // Define default args
+    args: {
+        title: 'Automations',
+        subTitle: 'Create & manage automations',
+        variant: 'pageHeader',
+    },
+} satisfies Meta<typeof PageTitle>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+// Basic story is what is displayed 1st in storybook
+// Pass props to this so that it can be customized via the UI props panel
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <PageTitle {...props} />,
+};
+
+export const withLink = () => (
+    <PageTitle
+        title="Automations"
+        subTitle={
+            <>
+                Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas aliquet nulla id felis vehicula, et
+                posuere dui dapibus. <a href="/">Nullam rhoncus massa non tortor convallis</a>, in blandit turpis
+                rutrum. Morbi tempus velit mauris, at mattis metus mattis sed. Nunc molestie efficitur lectus, vel
+                mollis eros.
+            </>
+        }
+    />
+);
+
+export const sectionHeader = () => (
+    <PageTitle title="Section header title" subTitle="Section header subtitle text" variant="sectionHeader" />
+);
diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.tsx b/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.tsx
new file mode 100644
index 0000000000000..3dcf42ff2fc0e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.tsx
@@ -0,0 +1,17 @@
+import React from 'react';
+import { PageTitleProps } from './types';
+import { Container, SubTitle, Title } from './components';
+import { Pill } from '../Pills';
+
+export const PageTitle = ({ title, subTitle, pillLabel, variant = 'pageHeader' }: PageTitleProps) => {
+    return (
+        <Container>
+            <Title variant={variant}>
+                {title}
+                {pillLabel ? <Pill label={pillLabel} size="sm" clickable={false} /> : null}
+            </Title>
+
+            {subTitle ? <SubTitle variant={variant}>{subTitle}</SubTitle> : null}
+        </Container>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/components.ts b/datahub-web-react/src/alchemy-components/components/PageTitle/components.ts
new file mode 100644
index 0000000000000..328323434e040
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/PageTitle/components.ts
@@ -0,0 +1,52 @@
+import styled from 'styled-components';
+import { typography, colors } from '@components/theme';
+import { getHeaderSubtitleStyles, getHeaderTitleStyles } from './utils';
+
+// Text Styles
+const titleStyles = {
+    display: 'flex',
+    alignItems: 'center',
+    gap: 8,
+    fontWeight: typography.fontWeights.bold,
+    color: colors.gray[600],
+};
+
+const subTitleStyles = {
+    fontWeight: typography.fontWeights.normal,
+    color: colors.gray[1700],
+};
+
+// Default styles
+const baseStyles = {
+    fontFamily: typography.fonts.body,
+    margin: 0,
+
+    '& a': {
+        color: colors.violet[400],
+        textDecoration: 'none',
+        transition: 'color 0.15s ease',
+
+        '&:hover': {
+            color: colors.violet[500],
+        },
+    },
+};
+
+export const Container = styled.div`
+    display: flex;
+    flex-direction: column;
+    align-items: start;
+    justify-content: start;
+`;
+
+export const Title = styled.div<{ variant: string }>(({ variant }) => ({
+    ...baseStyles,
+    ...titleStyles,
+    ...getHeaderTitleStyles(variant),
+}));
+
+export const SubTitle = styled.div<{ variant: string }>(({ variant }) => ({
+    ...baseStyles,
+    ...subTitleStyles,
+    ...getHeaderSubtitleStyles(variant),
+}));
diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/index.ts b/datahub-web-react/src/alchemy-components/components/PageTitle/index.ts
new file mode 100644
index 0000000000000..2888306f7c9a6
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/PageTitle/index.ts
@@ -0,0 +1 @@
+export { PageTitle } from './PageTitle';
diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/types.ts b/datahub-web-react/src/alchemy-components/components/PageTitle/types.ts
new file mode 100644
index 0000000000000..fb1e207d0bbd7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/PageTitle/types.ts
@@ -0,0 +1,8 @@
+import React from 'react';
+
+export interface PageTitleProps {
+    title: string;
+    subTitle?: string | React.ReactNode;
+    pillLabel?: string;
+    variant?: 'pageHeader' | 'sectionHeader';
+}
diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/utils.ts b/datahub-web-react/src/alchemy-components/components/PageTitle/utils.ts
new file mode 100644
index 0000000000000..fe6d18688f31f
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/PageTitle/utils.ts
@@ -0,0 +1,27 @@
+import { typography } from '@components/theme';
+
+export const getHeaderTitleStyles = (variant) => {
+    if (variant === 'sectionHeader') {
+        return {
+            fontSize: typography.fontSizes.lg,
+            lineHeight: typography.lineHeights.lg,
+        };
+    }
+    return {
+        fontSize: typography.fontSizes['3xl'],
+        lineHeight: typography.lineHeights['3xl'],
+    };
+};
+
+export const getHeaderSubtitleStyles = (variant) => {
+    if (variant === 'sectionHeader') {
+        return {
+            fontSize: typography.fontSizes.md,
+            lineHeight: typography.lineHeights.md,
+        };
+    }
+    return {
+        fontSize: typography.fontSizes.lg,
+        lineHeight: typography.lineHeights.lg,
+    };
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Pills/Pill.stories.tsx b/datahub-web-react/src/alchemy-components/components/Pills/Pill.stories.tsx
new file mode 100644
index 0000000000000..d5cdffef6d6bd
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Pills/Pill.stories.tsx
@@ -0,0 +1,126 @@
+import React from 'react';
+
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import type { Meta, StoryObj } from '@storybook/react';
+
+import { GridList } from '@components/.docs/mdx-components';
+import { AVAILABLE_ICONS } from '../Icon';
+import { Pill, pillDefault } from './Pill';
+
+const meta = {
+    title: 'Components / Pill',
+    component: Pill,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.EXPERIMENTAL],
+        docs: {
+            subtitle: 'A component that is used to get pill',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        label: {
+            description: 'Label for the Pill.',
+            table: {
+                defaultValue: { summary: pillDefault.label },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        leftIcon: {
+            description: 'The icon to display in the Pill icon.',
+            type: 'string',
+            options: AVAILABLE_ICONS,
+            control: {
+                type: 'select',
+            },
+        },
+        rightIcon: {
+            description: 'The icon to display in the Pill icon.',
+            type: 'string',
+            options: AVAILABLE_ICONS,
+            control: {
+                type: 'select',
+            },
+        },
+        size: {
+            description: 'The size of the pill.',
+            options: ['sm', 'md', 'lg', 'xl'],
+            table: {
+                defaultValue: { summary: pillDefault.size },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        variant: {
+            description: 'The size of the Pill.',
+            options: ['filled', 'outline'],
+            table: {
+                defaultValue: { summary: pillDefault.variant },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        colorScheme: {
+            description: 'The color of the Pill.',
+            options: ['violet', 'green', 'red', 'blue', 'gray'],
+            table: {
+                defaultValue: { summary: pillDefault.color },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+    },
+
+    // Define defaults
+    args: {
+        label: pillDefault.label,
+        leftIcon: pillDefault.leftIcon,
+        rightIcon: pillDefault.rightIcon,
+        size: pillDefault.size,
+        variant: pillDefault.variant,
+    },
+} satisfies Meta<typeof Pill>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Pill {...props} />,
+};
+
+export const sizes = () => (
+    <GridList>
+        <Pill label="Default" />
+        <Pill label="small" size="sm" />
+        <Pill label="large" size="lg" />
+    </GridList>
+);
+
+export const colors = () => (
+    <GridList>
+        <Pill label="Default" />
+        <Pill label="Violet" colorScheme="violet" />
+        <Pill label="Green" colorScheme="green" />
+        <Pill label="Red" colorScheme="red" />
+        <Pill label="Blue" colorScheme="blue" />
+        <Pill label="Gray" colorScheme="gray" />
+    </GridList>
+);
+
+export const withIcon = () => (
+    <GridList>
+        <Pill label="left icon" leftIcon="AutoMode" />
+        <Pill label="right icon" rightIcon="Close" />
+        <Pill label="both icon" leftIcon="AutoMode" rightIcon="Close" />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Pills/Pill.tsx b/datahub-web-react/src/alchemy-components/components/Pills/Pill.tsx
new file mode 100644
index 0000000000000..898ec89fce595
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Pills/Pill.tsx
@@ -0,0 +1,42 @@
+import { Icon } from '@components';
+import React from 'react';
+
+import { PillContainer, PillText } from './components';
+import { PillProps } from './types';
+
+export const pillDefault: PillProps = {
+    label: 'Label',
+    size: 'md',
+    variant: 'filled',
+    clickable: true,
+};
+
+export function Pill({
+    label = pillDefault.label,
+    size = pillDefault.size,
+    leftIcon,
+    rightIcon,
+    colorScheme,
+    variant = pillDefault.variant,
+    clickable = pillDefault.clickable,
+    id,
+    onClickRightIcon,
+    onClickLeftIcon,
+    onPillClick,
+}: PillProps) {
+    return (
+        <PillContainer
+            variant={variant}
+            colorScheme={colorScheme}
+            size={size}
+            clickable={clickable}
+            id={id}
+            data-testid="pill-container"
+            onClick={onPillClick}
+        >
+            {leftIcon && <Icon icon={leftIcon} size={size} onClick={onClickLeftIcon} />}
+            <PillText>{label}</PillText>
+            {rightIcon && <Icon icon={rightIcon} size={size} onClick={onClickRightIcon} />}
+        </PillContainer>
+    );
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Pills/components.ts b/datahub-web-react/src/alchemy-components/components/Pills/components.ts
new file mode 100644
index 0000000000000..79734561a92da
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Pills/components.ts
@@ -0,0 +1,33 @@
+import { spacing } from '@components/theme';
+import styled from 'styled-components';
+
+import { PillStyleProps } from './types';
+import { getPillStyle } from './utils';
+
+export const PillContainer = styled.div(
+    // Dynamic styles
+    (props: PillStyleProps) => ({ ...getPillStyle(props as PillStyleProps) }),
+    {
+        // Base root styles
+        display: 'inline-flex',
+        alignItems: 'center',
+        gap: spacing.xxsm,
+        cursor: 'pointer',
+        padding: '0px 8px',
+        borderRadius: '200px',
+        maxWidth: '100%',
+
+        // Base Disabled styles
+        '&:disabled': {
+            cursor: 'not-allowed',
+        },
+    },
+);
+
+export const PillText = styled.span({
+    maxWidth: '100%',
+    display: 'block',
+    whiteSpace: 'nowrap',
+    overflow: 'hidden',
+    textOverflow: 'ellipsis',
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Pills/index.ts b/datahub-web-react/src/alchemy-components/components/Pills/index.ts
new file mode 100644
index 0000000000000..85a76193db267
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Pills/index.ts
@@ -0,0 +1 @@
+export { Pill } from './Pill';
diff --git a/datahub-web-react/src/alchemy-components/components/Pills/types.ts b/datahub-web-react/src/alchemy-components/components/Pills/types.ts
new file mode 100644
index 0000000000000..17d4d12465e1e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Pills/types.ts
@@ -0,0 +1,18 @@
+import { ColorOptions, SizeOptions, VariantOptions } from '@src/alchemy-components/theme/config';
+import { HTMLAttributes } from 'react';
+
+export interface PillStyleProps {
+    colorScheme?: ColorOptions; // need to keep colorScheme because HTMLAttributes also have color property
+    variant?: VariantOptions;
+    size?: SizeOptions;
+    clickable?: boolean;
+}
+
+export interface PillProps extends HTMLAttributes<HTMLElement>, PillStyleProps {
+    label: string;
+    rightIcon?: string;
+    leftIcon?: string;
+    onClickRightIcon?: (e: React.MouseEvent<HTMLElement, MouseEvent>) => void;
+    onClickLeftIcon?: (e: React.MouseEvent<HTMLElement, MouseEvent>) => void;
+    onPillClick?: (e: React.MouseEvent<HTMLElement, MouseEvent>) => void;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Pills/utils.ts b/datahub-web-react/src/alchemy-components/components/Pills/utils.ts
new file mode 100644
index 0000000000000..832bf95640982
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Pills/utils.ts
@@ -0,0 +1,147 @@
+import { colors, typography } from '@src/alchemy-components/theme';
+import { getColor, getFontSize } from '@src/alchemy-components/theme/utils';
+import { PillStyleProps } from './types';
+
+// Utility function to get color styles for pill - does not generate CSS
+const getPillColorStyles = (variant, color) => {
+    const defaultStyles = {
+        bgColor: getColor(color, 100),
+        hoverBgColor: getColor('gray', 100),
+        borderColor: '',
+        activeBorderColor: getColor('violet', 500),
+        textColor: getColor(color, 600),
+    };
+
+    const colorOverrides = {
+        violet: {
+            textColor: getColor(color, 500),
+            bgColor: getColor('gray', 1000),
+            borderColor: 'transparent',
+            hoverBgColor: getColor(color, 100),
+            activeBorderColor: getColor(color, 500),
+        },
+        blue: {
+            textColor: getColor(color, 1000),
+            bgColor: getColor('gray', 1100),
+            borderColor: 'transparent',
+            hoverBgColor: getColor(color, 1100),
+            activeBorderColor: getColor(color, 1000),
+        },
+        red: {
+            textColor: getColor(color, 1000),
+            bgColor: getColor('gray', 1200),
+            hoverBgColor: getColor(color, 1100),
+            activeBorderColor: getColor(color, 1000),
+        },
+        green: {
+            textColor: getColor(color, 1000),
+            bgColor: getColor('gray', 1300),
+            hoverBgColor: getColor(color, 1100),
+            activeBorderColor: getColor(color, 1000),
+        },
+        yellow: {
+            textColor: getColor(color, 1000),
+            bgColor: getColor('gray', 1400),
+            hoverBgColor: getColor(color, 1100),
+            activeBorderColor: getColor(color, 1000),
+        },
+    };
+
+    const styles = colorOverrides[color] || defaultStyles;
+
+    if (variant === 'outline') {
+        return {
+            bgColor: colors.transparent,
+            borderColor: getColor('gray', 1400),
+            textColor: getColor(color, 600),
+        };
+    }
+
+    return styles;
+};
+
+// Generate variant styles for pill
+const getPillVariantStyles = (variant, colorStyles) =>
+    ({
+        filled: {
+            backgroundColor: colorStyles.bgColor,
+            border: `1px solid transparent`,
+            color: colorStyles.textColor,
+            '&:hover': {
+                backgroundColor: colorStyles.hoverBgColor,
+            },
+        },
+        outline: {
+            backgroundColor: 'transparent',
+            border: `1px solid ${colorStyles.borderColor}`,
+            color: colorStyles.textColor,
+            '&:hover': {
+                backgroundColor: colorStyles.hoverBgColor,
+                border: `1px solid transparent`,
+            },
+            '&:disabled': {
+                border: `1px solid transparent`,
+            },
+        },
+        text: {
+            color: colorStyles.textColor,
+        },
+    }[variant]);
+
+// Generate font styles for pill
+const getPillFontStyles = (size) => {
+    const baseFontStyles = {
+        fontFamily: typography.fonts.body,
+        fontWeight: typography.fontWeights.normal,
+        lineHeight: typography.lineHeights.none,
+    };
+
+    const sizeMap = {
+        xs: { fontSize: getFontSize(size), lineHeight: '16px' },
+        sm: { fontSize: getFontSize(size), lineHeight: '22px' },
+        md: { fontSize: getFontSize(size), lineHeight: '24px' },
+        lg: { fontSize: getFontSize(size), lineHeight: '30px' },
+        xl: { fontSize: getFontSize(size), lineHeight: '34px' },
+    };
+
+    return {
+        ...baseFontStyles,
+        ...sizeMap[size],
+    };
+};
+
+// Generate active styles for pill
+const getPillActiveStyles = (styleColors) => ({
+    borderColor: styleColors.activeBorderColor,
+});
+
+/*
+ * Main function to generate styles for pill
+ */
+export const getPillStyle = (props: PillStyleProps) => {
+    const { variant, colorScheme = 'gray', size, clickable = true } = props;
+
+    // Get map of colors
+    const colorStyles = getPillColorStyles(variant, colorScheme);
+
+    // Define styles for pill
+    let styles = {
+        ...getPillVariantStyles(variant, colorStyles),
+        ...getPillFontStyles(size),
+        '&:focus': {
+            ...getPillActiveStyles(colorStyles),
+            outline: 'none', // Remove default browser focus outline if needed
+        },
+        '&:active': {
+            ...getPillActiveStyles(colorStyles),
+        },
+    };
+    if (!clickable) {
+        styles = {
+            ...styles,
+            pointerEvents: 'none',
+        };
+    }
+
+    return styles;
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Popover/Popover.tsx b/datahub-web-react/src/alchemy-components/components/Popover/Popover.tsx
new file mode 100644
index 0000000000000..8f6ca61976b20
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Popover/Popover.tsx
@@ -0,0 +1,6 @@
+import { Popover, PopoverProps } from 'antd';
+import * as React from 'react';
+
+export default function DataHubPopover(props: PopoverProps & React.RefAttributes<unknown>) {
+    return <Popover {...props} showArrow={false} />;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Popover/index.ts b/datahub-web-react/src/alchemy-components/components/Popover/index.ts
new file mode 100644
index 0000000000000..02df6c38e8c4e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Popover/index.ts
@@ -0,0 +1 @@
+export { default as Popover } from './Popover';
diff --git a/datahub-web-react/src/alchemy-components/components/Radio/Radio.stories.tsx b/datahub-web-react/src/alchemy-components/components/Radio/Radio.stories.tsx
new file mode 100644
index 0000000000000..cb3116d7b8941
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Radio/Radio.stories.tsx
@@ -0,0 +1,136 @@
+import React from 'react';
+import type { Meta, StoryObj } from '@storybook/react';
+import { GridList } from '@components/.docs/mdx-components';
+import { Radio, radioDefaults, RadioGroup } from './Radio';
+import { Heading } from '../Heading';
+import { RadioProps } from './types';
+
+const MOCK_RADIOS: RadioProps[] = [
+    {
+        label: 'Label 1',
+        error: '',
+        isChecked: false,
+        isDisabled: false,
+        isIntermediate: false,
+        isRequired: false,
+    },
+    {
+        label: 'Label 2',
+        error: '',
+        isChecked: false,
+        isDisabled: false,
+        isIntermediate: false,
+        isRequired: false,
+    },
+    {
+        label: 'Label 3',
+        error: '',
+        isChecked: false,
+        isDisabled: false,
+        isIntermediate: false,
+        isRequired: false,
+    },
+];
+
+const meta = {
+    title: 'Forms / Radio',
+    component: Radio,
+    parameters: {
+        layout: 'centered',
+        docs: {
+            subtitle: 'A component that is used to get user input in the state of a radio button.',
+        },
+    },
+    argTypes: {
+        label: {
+            description: 'Label for the Radio.',
+            table: {
+                defaultValue: { summary: radioDefaults.label },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        error: {
+            description: 'Enforce error state on the Radio.',
+            table: {
+                defaultValue: { summary: radioDefaults.error },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        isChecked: {
+            description: 'Whether the Radio is checked.',
+            table: {
+                defaultValue: { summary: radioDefaults?.isChecked?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isDisabled: {
+            description: 'Whether the Radio is in disabled state.',
+            table: {
+                defaultValue: { summary: radioDefaults?.isDisabled?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isRequired: {
+            description: 'Whether the Radio is a required field.',
+            table: {
+                defaultValue: { summary: radioDefaults?.isRequired?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+    },
+    args: {
+        label: radioDefaults.label,
+        error: radioDefaults.error,
+        isChecked: radioDefaults.isChecked,
+        isDisabled: radioDefaults.isDisabled,
+        isRequired: radioDefaults.isRequired,
+    },
+} satisfies Meta<typeof Radio>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Radio {...props} />,
+};
+
+export const states = () => (
+    <GridList>
+        <Radio label="Default" />
+        <Radio label="Checked" isChecked />
+        <Radio label="Error" isChecked error="Error" />
+        <Radio label="Required" isChecked isRequired />
+    </GridList>
+);
+
+export const disabledStates = () => (
+    <GridList>
+        <Radio label="Default" isDisabled />
+        <Radio label="Checked" isChecked isDisabled />
+    </GridList>
+);
+
+export const radioGroups = () => (
+    <GridList isVertical>
+        <div>
+            <Heading>Horizontal Radio Group</Heading>
+            <RadioGroup radios={MOCK_RADIOS} />
+        </div>
+        <div>
+            <Heading>Vertical Radio Group</Heading>
+            <RadioGroup isVertical radios={MOCK_RADIOS} />
+        </div>
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Radio/Radio.tsx b/datahub-web-react/src/alchemy-components/components/Radio/Radio.tsx
new file mode 100644
index 0000000000000..592c10ec88de8
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Radio/Radio.tsx
@@ -0,0 +1,89 @@
+import React, { useEffect, useState } from 'react';
+import { RadioGroupProps, RadioProps } from './types';
+import {
+    RadioWrapper,
+    Checkmark,
+    HiddenInput,
+    Label,
+    Required,
+    RadioLabel,
+    RadioBase,
+    RadioGroupContainer,
+} from './components';
+
+export const radioDefaults = {
+    label: 'Label',
+    error: '',
+    isChecked: false,
+    isDisabled: false,
+    isRequired: false,
+    isVertical: false,
+    setIsChecked: () => {},
+};
+
+export const Radio = ({
+    label = radioDefaults.label,
+    error = radioDefaults.error,
+    isChecked = radioDefaults.isChecked,
+    isDisabled = radioDefaults.isDisabled,
+    isRequired = radioDefaults.isRequired,
+    setIsChecked = radioDefaults.setIsChecked,
+    ...props
+}: RadioProps) => {
+    const [checked, setChecked] = useState(isChecked || false);
+
+    useEffect(() => {
+        setChecked(isChecked || false);
+    }, [isChecked]);
+
+    const id = props.id || `checkbox-${label}`;
+
+    return (
+        <RadioWrapper disabled={isDisabled} error={error}>
+            <RadioBase>
+                <HiddenInput
+                    type="radio"
+                    id={label}
+                    value={label}
+                    checked={checked}
+                    disabled={isDisabled}
+                    onChange={() => {
+                        setChecked(true);
+                        setIsChecked?.(true);
+                    }}
+                    aria-label={label}
+                    aria-labelledby={id}
+                    aria-checked={checked}
+                    {...props}
+                />
+                <Checkmark checked={checked} disabled={isDisabled} error={error} />
+            </RadioBase>
+            {label && (
+                <RadioLabel>
+                    <Label onClick={() => setChecked(true)}>
+                        {label} {isRequired && <Required>*</Required>}
+                    </Label>
+                </RadioLabel>
+            )}
+        </RadioWrapper>
+    );
+};
+
+export const RadioGroup = ({ isVertical, radios }: RadioGroupProps) => {
+    if (!radios.length) {
+        return <></>;
+    }
+
+    return (
+        <RadioGroupContainer isVertical={isVertical}>
+            {radios.map((checkbox) => {
+                const props = { ...checkbox };
+                return (
+                    <React.Fragment key={checkbox.label}>
+                        <Radio {...props} />
+                    </React.Fragment>
+                );
+            })}
+        </RadioGroupContainer>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Radio/components.ts b/datahub-web-react/src/alchemy-components/components/Radio/components.ts
new file mode 100644
index 0000000000000..027971be17958
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Radio/components.ts
@@ -0,0 +1,83 @@
+import { borders, colors, radius, spacing } from '@components/theme';
+import styled from 'styled-components';
+import { formLabelTextStyles } from '../commonStyles';
+import { getRadioBorderColor, getRadioCheckmarkColor } from './utils';
+
+export const RadioWrapper = styled.div<{ disabled: boolean; error: string }>(({ disabled, error }) => ({
+    position: 'relative',
+    margin: '20px',
+    width: '20px',
+    height: '20px',
+    border: `${borders['2px']} ${getRadioBorderColor(disabled, error)}`,
+    backgroundColor: colors.white,
+    borderRadius: radius.full,
+    display: 'flex',
+    justifyContent: 'flex-start',
+    alignItems: 'center',
+    marginRight: '40px',
+    cursor: !disabled ? 'pointer' : 'none',
+    transition: 'border 0.3s ease, outline 0.3s ease',
+    '&:hover': {
+        border: `${borders['2px']} ${!disabled && !error ? colors.violet[500] : getRadioBorderColor(disabled, error)}`,
+        outline: !disabled && !error ? `${borders['2px']} ${colors.gray[200]}` : 'none',
+    },
+}));
+
+export const RadioBase = styled.div({});
+
+export const Label = styled.div({
+    ...formLabelTextStyles,
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+});
+
+export const RadioLabel = styled.div({
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+});
+
+export const Required = styled.span({
+    color: colors.red[500],
+    marginLeft: spacing.xxsm,
+});
+
+export const RadioHoverState = styled.div({
+    border: `${borders['2px']} ${colors.violet[500]}`,
+    width: 'calc(100% - -3px)',
+    height: 'calc(100% - -3px)',
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+    borderRadius: radius.full,
+});
+
+export const Checkmark = styled.div<{ checked: boolean; disabled: boolean; error: string }>(
+    ({ checked, disabled, error }) => ({
+        width: 'calc(100% - 6px)',
+        height: 'calc(100% - 6px)',
+        borderRadius: radius.full,
+        background: getRadioCheckmarkColor(checked, disabled, error),
+        display: checked ? 'inline-block' : 'none',
+        position: 'absolute',
+        top: '50%',
+        left: '50%',
+        transform: 'translate(-50%, -50%)',
+    }),
+);
+
+export const HiddenInput = styled.input<{ checked: boolean }>({
+    opacity: 0,
+    width: '20px',
+    height: '20px',
+});
+
+export const RadioGroupContainer = styled.div<{ isVertical?: boolean }>(({ isVertical }) => ({
+    display: 'flex',
+    flexDirection: isVertical ? 'column' : 'row',
+    justifyContent: 'center',
+    alignItems: 'center',
+    gap: !isVertical ? spacing.md : spacing.none,
+    margin: !isVertical ? spacing.xxsm : spacing.none,
+}));
diff --git a/datahub-web-react/src/alchemy-components/components/Radio/types.ts b/datahub-web-react/src/alchemy-components/components/Radio/types.ts
new file mode 100644
index 0000000000000..59fd15654f916
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Radio/types.ts
@@ -0,0 +1,16 @@
+import { InputHTMLAttributes } from 'react';
+
+export interface RadioProps extends InputHTMLAttributes<HTMLInputElement> {
+    label?: string;
+    error?: string;
+    isChecked?: boolean;
+    setIsChecked?: React.Dispatch<React.SetStateAction<boolean>>;
+    isDisabled?: boolean;
+    isIntermediate?: boolean;
+    isRequired?: boolean;
+}
+
+export interface RadioGroupProps {
+    isVertical?: boolean;
+    radios: RadioProps[];
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Radio/utils.ts b/datahub-web-react/src/alchemy-components/components/Radio/utils.ts
new file mode 100644
index 0000000000000..ed9dcc35d303b
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Radio/utils.ts
@@ -0,0 +1,27 @@
+import { colors } from '@components/theme';
+
+const radioBorderColors = {
+    default: colors.gray[400],
+    disabled: colors.gray[300],
+    error: colors.red[500],
+};
+
+const radioCheckmarkColors = {
+    default: colors.white,
+    disabled: colors.gray[300],
+    checked: colors.violet[500],
+    error: colors.red[500],
+};
+
+export function getRadioBorderColor(disabled: boolean, error: string) {
+    if (disabled) return radioBorderColors.disabled;
+    if (error) return radioCheckmarkColors.error;
+    return radioBorderColors.default;
+}
+
+export function getRadioCheckmarkColor(checked: boolean, disabled: boolean, error: string) {
+    if (disabled) return radioCheckmarkColors.disabled;
+    if (error) return radioCheckmarkColors.error;
+    if (checked) return radioCheckmarkColors.checked;
+    return radioCheckmarkColors.default;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx
new file mode 100644
index 0000000000000..9ae34356a71d6
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx
@@ -0,0 +1,99 @@
+import React, { useState } from 'react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import type { Meta, StoryObj } from '@storybook/react';
+import { GridList } from '@components/.docs/mdx-components';
+import { SearchBar, searchBarDefaults } from './SearchBar';
+import { SearchBarProps } from './types';
+
+const meta = {
+    title: 'Components / Search Bar',
+    component: SearchBar,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'A component that is used to get search bar',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        placeholder: {
+            description: 'Placeholder of search bar.',
+            table: {
+                defaultValue: { summary: searchBarDefaults.placeholder },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        value: {
+            description: 'Value of the search bar.',
+            table: {
+                defaultValue: { summary: searchBarDefaults.value },
+            },
+            control: false,
+        },
+        width: {
+            description: 'Width of the search bar.',
+            table: {
+                defaultValue: { summary: searchBarDefaults.width },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        allowClear: {
+            description: 'Whether clear button should be present.',
+            table: {
+                defaultValue: { summary: searchBarDefaults.allowClear?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        onChange: {
+            description: 'On change function for the search bar.',
+        },
+    },
+
+    // Define defaults
+    args: {
+        placeholder: searchBarDefaults.placeholder,
+        value: searchBarDefaults.value,
+        allowClear: searchBarDefaults.allowClear,
+        width: searchBarDefaults.width,
+    },
+} satisfies Meta<typeof SearchBar>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+const SandboxWrapper = (props: SearchBarProps) => {
+    const [value, setValue] = useState('');
+
+    const handleChange = (newValue: string) => {
+        setValue(newValue);
+    };
+
+    return <SearchBar {...props} value={value} onChange={handleChange} />;
+};
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => {
+        return <SandboxWrapper {...props} />;
+    },
+};
+
+export const customWidths = () => (
+    <GridList isVertical>
+        <SearchBar width="500px" />
+        <SearchBar width="300px" />
+        <SearchBar width="200px" />
+        <SearchBar width="150px" />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx
new file mode 100644
index 0000000000000..f39f761058d8c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx
@@ -0,0 +1,30 @@
+import { SearchOutlined } from '@ant-design/icons';
+import React from 'react';
+import { StyledSearchBar } from './components';
+import { SearchBarProps } from './types';
+
+export const searchBarDefaults: SearchBarProps = {
+    placeholder: 'Search..',
+    value: '',
+    width: '272px',
+    allowClear: true,
+};
+
+export const SearchBar = ({
+    placeholder = searchBarDefaults.placeholder,
+    value = searchBarDefaults.value,
+    width = searchBarDefaults.width,
+    allowClear = searchBarDefaults.allowClear,
+    onChange,
+}: SearchBarProps) => {
+    return (
+        <StyledSearchBar
+            placeholder={placeholder}
+            onChange={(e) => onChange?.(e.target.value)}
+            value={value}
+            prefix={<SearchOutlined />}
+            allowClear={allowClear}
+            $width={width}
+        />
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts
new file mode 100644
index 0000000000000..7045801ddf092
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts
@@ -0,0 +1,33 @@
+import { colors, typography } from '@src/alchemy-components/theme';
+import { Input } from 'antd';
+import styled from 'styled-components';
+
+export const StyledSearchBar = styled(Input)<{ $width?: string }>`
+    height: 40px;
+    width: ${(props) => props.$width};
+    display: flex;
+    align-items: center;
+    border-radius: 8px;
+
+    input {
+        color: ${colors.gray[600]};
+        font-size: ${typography.fontSizes.md} !important;
+    }
+
+    .ant-input-prefix {
+        width: 20px;
+        color: ${colors.gray[1800]};
+
+        svg {
+            height: 16px;
+            width: 16px;
+        }
+    }
+
+    &:hover,
+    &:focus,
+    &:focus-within {
+        border-color: ${colors.violet[300]} !important;
+        box-shadow: none !important;
+    }
+`;
diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts
new file mode 100644
index 0000000000000..8c1933163b29f
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts
@@ -0,0 +1 @@
+export { SearchBar } from './SearchBar';
diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts
new file mode 100644
index 0000000000000..04ac218dc2cc6
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts
@@ -0,0 +1,7 @@
+export interface SearchBarProps {
+    placeholder?: string;
+    value?: string;
+    width?: string;
+    onChange?: (value: string) => void;
+    allowClear?: boolean;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Select/BasicSelect.tsx b/datahub-web-react/src/alchemy-components/components/Select/BasicSelect.tsx
new file mode 100644
index 0000000000000..b49159ba38a75
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/BasicSelect.tsx
@@ -0,0 +1,339 @@
+import { Button, Icon, Pill, Text } from '@components';
+import { isEqual } from 'lodash';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import {
+    ActionButtonsContainer,
+    Container,
+    DescriptionContainer,
+    Dropdown,
+    FooterBase,
+    LabelContainer,
+    LabelsWrapper,
+    OptionContainer,
+    OptionLabel,
+    OptionList,
+    Placeholder,
+    SearchIcon,
+    SearchInput,
+    SearchInputContainer,
+    SelectAllOption,
+    SelectBase,
+    SelectLabel,
+    SelectValue,
+    StyledCancelButton,
+    StyledCheckbox,
+    StyledClearButton,
+} from './components';
+import { ActionButtonsProps, SelectLabelDisplayProps, SelectOption, SelectProps } from './types';
+import { getFooterButtonSize } from './utils';
+
+const SelectLabelDisplay = ({
+    selectedValues,
+    options,
+    placeholder,
+    isMultiSelect,
+    removeOption,
+    disabledValues,
+    showDescriptions,
+}: SelectLabelDisplayProps) => {
+    const selectedOptions = options.filter((opt) => selectedValues.includes(opt.value));
+    return (
+        <LabelsWrapper>
+            {!!selectedOptions.length &&
+                isMultiSelect &&
+                selectedOptions.map((o) => {
+                    const isDisabled = disabledValues?.includes(o.value);
+                    return (
+                        <Pill
+                            label={o.label}
+                            rightIcon={!isDisabled ? 'Close' : ''}
+                            size="sm"
+                            key={o.value}
+                            onClickRightIcon={(e) => {
+                                e.stopPropagation();
+                                removeOption?.(o);
+                            }}
+                            clickable={!isDisabled}
+                        />
+                    );
+                })}
+            {!selectedValues.length && <Placeholder>{placeholder}</Placeholder>}
+            {!isMultiSelect && (
+                <>
+                    <SelectValue>{selectedOptions[0]?.label}</SelectValue>
+                    {showDescriptions && !!selectedValues.length && (
+                        <DescriptionContainer>{selectedOptions[0]?.description}</DescriptionContainer>
+                    )}
+                </>
+            )}
+        </LabelsWrapper>
+    );
+};
+
+const SelectActionButtons = ({
+    selectedValues,
+    isOpen,
+    isDisabled,
+    isReadOnly,
+    showClear,
+    handleClearSelection,
+    fontSize = 'md',
+}: ActionButtonsProps) => {
+    return (
+        <ActionButtonsContainer>
+            {showClear && selectedValues.length > 0 && !isDisabled && !isReadOnly && (
+                <StyledClearButton icon="Close" isCircle onClick={handleClearSelection} size={fontSize} />
+            )}
+            <Icon icon="ChevronLeft" rotate={isOpen ? '90' : '270'} size="xl" color="gray" />
+        </ActionButtonsContainer>
+    );
+};
+
+// Updated main component
+export const selectDefaults: SelectProps = {
+    options: [],
+    label: '',
+    size: 'md',
+    showSearch: false,
+    isDisabled: false,
+    isReadOnly: false,
+    isRequired: false,
+    isMultiSelect: false,
+    showClear: false,
+    placeholder: 'Select an option',
+    showSelectAll: false,
+    selectAllLabel: 'Select All',
+    showDescriptions: false,
+};
+
+export const BasicSelect = ({
+    options = selectDefaults.options,
+    label = selectDefaults.label,
+    values = [],
+    onCancel,
+    onUpdate,
+    showSearch = selectDefaults.showSearch,
+    isDisabled = selectDefaults.isDisabled,
+    isReadOnly = selectDefaults.isReadOnly,
+    isRequired = selectDefaults.isRequired,
+    showClear = selectDefaults.showClear,
+    size = selectDefaults.size,
+    isMultiSelect = selectDefaults.isMultiSelect,
+    placeholder = selectDefaults.placeholder,
+    disabledValues = [],
+    showSelectAll = selectDefaults.showSelectAll,
+    selectAllLabel = selectDefaults.selectAllLabel,
+    showDescriptions = selectDefaults.showDescriptions,
+    ...props
+}: SelectProps) => {
+    const [searchQuery, setSearchQuery] = useState('');
+    const [isOpen, setIsOpen] = useState(false);
+    const [selectedValues, setSelectedValues] = useState<string[]>(values);
+    const [tempValues, setTempValues] = useState<string[]>(values);
+    const selectRef = useRef<HTMLDivElement>(null);
+    const [areAllSelected, setAreAllSelected] = useState(false);
+
+    useEffect(() => {
+        if (values?.length > 0 && !isEqual(selectedValues, values)) {
+            setSelectedValues(values);
+        }
+    }, [values, selectedValues]);
+
+    useEffect(() => {
+        setAreAllSelected(tempValues.length === options.length);
+    }, [options, tempValues]);
+
+    const filteredOptions = useMemo(
+        () => options.filter((option) => option.label.toLowerCase().includes(searchQuery.toLowerCase())),
+        [options, searchQuery],
+    );
+
+    const handleDocumentClick = useCallback((e: MouseEvent) => {
+        if (selectRef.current && !selectRef.current.contains(e.target as Node)) {
+            setIsOpen(false);
+        }
+    }, []);
+
+    useEffect(() => {
+        document.addEventListener('click', handleDocumentClick);
+        return () => {
+            document.removeEventListener('click', handleDocumentClick);
+        };
+    }, [handleDocumentClick]);
+
+    const handleSelectClick = useCallback(() => {
+        if (!isDisabled && !isReadOnly) {
+            setTempValues(selectedValues);
+            setIsOpen((prev) => !prev);
+        }
+    }, [isDisabled, isReadOnly, selectedValues]);
+
+    const handleOptionChange = useCallback(
+        (option: SelectOption) => {
+            const updatedValues = tempValues.includes(option.value)
+                ? tempValues.filter((val) => val !== option.value)
+                : [...tempValues, option.value];
+
+            setTempValues(isMultiSelect ? updatedValues : [option.value]);
+        },
+        [tempValues, isMultiSelect],
+    );
+
+    const removeOption = useCallback(
+        (option: SelectOption) => {
+            const updatedValues = selectedValues.filter((val) => val !== option.value);
+            setSelectedValues(updatedValues);
+        },
+        [selectedValues],
+    );
+
+    const handleUpdateClick = useCallback(() => {
+        setSelectedValues(tempValues);
+        setIsOpen(false);
+        if (onUpdate) {
+            onUpdate(tempValues);
+        }
+    }, [tempValues, onUpdate]);
+
+    const handleCancelClick = useCallback(() => {
+        setIsOpen(false);
+        setTempValues(selectedValues);
+        if (onCancel) {
+            onCancel();
+        }
+    }, [selectedValues, onCancel]);
+
+    const handleClearSelection = useCallback(() => {
+        setSelectedValues([]);
+        setAreAllSelected(false);
+        setTempValues([]);
+        setIsOpen(false);
+        if (onUpdate) {
+            onUpdate([]);
+        }
+    }, [onUpdate]);
+
+    const handleSelectAll = () => {
+        if (areAllSelected) {
+            setTempValues([]);
+            onUpdate?.([]);
+        } else {
+            const allValues = options.map((option) => option.value);
+            setTempValues(allValues);
+            onUpdate?.(allValues);
+        }
+        setAreAllSelected(!areAllSelected);
+    };
+
+    return (
+        <Container ref={selectRef} size={size || 'md'} width={props.width}>
+            {label && <SelectLabel onClick={handleSelectClick}>{label}</SelectLabel>}
+            <SelectBase
+                isDisabled={isDisabled}
+                isReadOnly={isReadOnly}
+                isRequired={isRequired}
+                isOpen={isOpen}
+                onClick={handleSelectClick}
+                fontSize={size}
+                {...props}
+            >
+                <SelectLabelDisplay
+                    selectedValues={selectedValues}
+                    options={options}
+                    placeholder={placeholder || 'Select an option'}
+                    isMultiSelect={isMultiSelect}
+                    removeOption={removeOption}
+                    disabledValues={disabledValues}
+                    showDescriptions={showDescriptions}
+                />
+                <SelectActionButtons
+                    selectedValues={selectedValues}
+                    isOpen={isOpen}
+                    isDisabled={!!isDisabled}
+                    isReadOnly={!!isReadOnly}
+                    handleClearSelection={handleClearSelection}
+                    fontSize={size}
+                    showClear={!!showClear}
+                />
+            </SelectBase>
+            {isOpen && (
+                <Dropdown>
+                    {showSearch && (
+                        <SearchInputContainer>
+                            <SearchInput
+                                type="text"
+                                placeholder="Search…"
+                                value={searchQuery}
+                                onChange={(e) => setSearchQuery(e.target.value)}
+                                style={{ fontSize: size || 'md' }}
+                            />
+                            <SearchIcon icon="Search" size={size} color="gray" />
+                        </SearchInputContainer>
+                    )}
+                    <OptionList>
+                        {showSelectAll && isMultiSelect && (
+                            <SelectAllOption
+                                isSelected={areAllSelected}
+                                onClick={() => !(disabledValues.length === options.length) && handleSelectAll()}
+                                isDisabled={disabledValues.length === options.length}
+                            >
+                                <LabelContainer>
+                                    <span>{selectAllLabel}</span>
+                                    <StyledCheckbox
+                                        checked={areAllSelected}
+                                        disabled={disabledValues.length === options.length}
+                                    />
+                                </LabelContainer>
+                            </SelectAllOption>
+                        )}
+                        {filteredOptions.map((option) => (
+                            <OptionLabel
+                                key={option.value}
+                                onClick={() => !isMultiSelect && handleOptionChange(option)}
+                                isSelected={tempValues.includes(option.value)}
+                                isMultiSelect={isMultiSelect}
+                                isDisabled={disabledValues?.includes(option.value)}
+                            >
+                                {isMultiSelect ? (
+                                    <LabelContainer>
+                                        <span>{option.label}</span>
+                                        <StyledCheckbox
+                                            onClick={() => handleOptionChange(option)}
+                                            checked={tempValues.includes(option.value)}
+                                            disabled={disabledValues?.includes(option.value)}
+                                        />
+                                    </LabelContainer>
+                                ) : (
+                                    <OptionContainer>
+                                        <Text color="gray" weight="semiBold" size="md">
+                                            {option.label}
+                                        </Text>
+                                        {!!option.description && (
+                                            <Text color="gray" weight="normal" size="sm">
+                                                {option.description}
+                                            </Text>
+                                        )}
+                                    </OptionContainer>
+                                )}
+                            </OptionLabel>
+                        ))}
+                    </OptionList>
+                    <FooterBase>
+                        <StyledCancelButton
+                            onClick={handleCancelClick}
+                            variant="filled"
+                            size={getFooterButtonSize(size)}
+                        >
+                            Cancel
+                        </StyledCancelButton>
+                        <Button onClick={handleUpdateClick} size={getFooterButtonSize(size)}>
+                            Update
+                        </Button>
+                    </FooterBase>
+                </Dropdown>
+            )}
+        </Container>
+    );
+};
+
+export default BasicSelect;
diff --git a/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedOption.tsx b/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedOption.tsx
new file mode 100644
index 0000000000000..8a7d3670b2b1b
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedOption.tsx
@@ -0,0 +1,309 @@
+import React, { useState, useMemo, useEffect } from 'react';
+
+import { colors, Icon } from '@components';
+import theme from '@components/theme';
+import styled from 'styled-components';
+import { Checkbox } from 'antd';
+
+import { OptionLabel } from '../components';
+import { SelectOption } from './types';
+
+const ParentOption = styled.div`
+    display: flex;
+    align-items: center;
+`;
+
+const ChildOptions = styled.div`
+    padding-left: 20px;
+`;
+
+const StyledCheckbox = styled(Checkbox)<{ checked: boolean; indeterminate?: boolean }>`
+    .ant-checkbox-inner {
+        border: 1px solid ${colors.gray[300]} !important;
+        border-radius: 3px;
+    }
+    margin-left: auto;
+    ${(props) =>
+        props.checked &&
+        !props.indeterminate &&
+        `
+		.ant-checkbox-inner {
+			background-color: ${theme.semanticTokens.colors.primary};
+			border-color: ${theme.semanticTokens.colors.primary} !important;
+		}
+	`}
+    ${(props) =>
+        props.indeterminate &&
+        `
+		.ant-checkbox-inner {
+			&:after {
+				background-color: ${theme.semanticTokens.colors.primary};
+			}
+		}
+	`}
+    ${(props) =>
+        props.disabled &&
+        `
+		.ant-checkbox-inner {
+			background-color: ${colors.gray[200]} !important;
+		}
+	`}
+`;
+
+function getChildrenRecursively(
+    directChildren: SelectOption[],
+    parentValueToOptions: { [parentValue: string]: SelectOption[] },
+) {
+    const visitedParents = new Set<string>();
+    let allChildren: SelectOption[] = [];
+
+    function getChildren(parentValue: string) {
+        const newChildren = parentValueToOptions[parentValue] || [];
+        if (visitedParents.has(parentValue) || !newChildren.length) {
+            return;
+        }
+
+        visitedParents.add(parentValue);
+        allChildren = [...allChildren, ...newChildren];
+        newChildren.forEach((child) => getChildren(child.value || child.value));
+    }
+
+    directChildren.forEach((c) => getChildren(c.value || c.value));
+
+    return allChildren;
+}
+
+interface OptionProps {
+    option: SelectOption;
+    selectedOptions: SelectOption[];
+    parentValueToOptions: { [parentValue: string]: SelectOption[] };
+    areParentsSelectable: boolean;
+    handleOptionChange: (node: SelectOption) => void;
+    addOptions: (nodes: SelectOption[]) => void;
+    removeOptions: (nodes: SelectOption[]) => void;
+    loadData?: (node: SelectOption) => void;
+    isMultiSelect?: boolean;
+    isLoadingParentChildList?: boolean;
+    setSelectedOptions: React.Dispatch<React.SetStateAction<SelectOption[]>>;
+}
+
+export const NestedOption = ({
+    option,
+    selectedOptions,
+    parentValueToOptions,
+    handleOptionChange,
+    addOptions,
+    removeOptions,
+    loadData,
+    isMultiSelect,
+    areParentsSelectable,
+    isLoadingParentChildList,
+    setSelectedOptions,
+}: OptionProps) => {
+    const [autoSelectChildren, setAutoSelectChildren] = useState(false);
+    const [loadingParentUrns, setLoadingParentUrns] = useState<string[]>([]);
+    const [isOpen, setIsOpen] = useState(false);
+    const directChildren = useMemo(
+        () => parentValueToOptions[option.value] || [],
+        [parentValueToOptions, option.value],
+    );
+
+    const recursiveChildren = useMemo(
+        () => getChildrenRecursively(directChildren, parentValueToOptions),
+        [directChildren, parentValueToOptions],
+    );
+
+    const children = useMemo(() => [...directChildren, ...recursiveChildren], [directChildren, recursiveChildren]);
+    const selectableChildren = useMemo(
+        () => (areParentsSelectable ? children : children.filter((c) => !c.isParent)),
+        [areParentsSelectable, children],
+    );
+    const parentChildren = useMemo(() => children.filter((c) => c.isParent), [children]);
+
+    useEffect(() => {
+        if (autoSelectChildren && selectableChildren.length) {
+            addOptions(selectableChildren);
+            setAutoSelectChildren(false);
+        }
+    }, [autoSelectChildren, selectableChildren, addOptions]);
+
+    const areAllChildrenSelected = useMemo(
+        () => selectableChildren.every((child) => selectedOptions.find((o) => o.value === child.value)),
+        [selectableChildren, selectedOptions],
+    );
+
+    const areAnyChildrenSelected = useMemo(
+        () => selectableChildren.some((child) => selectedOptions.find((o) => o.value === child.value)),
+        [selectableChildren, selectedOptions],
+    );
+
+    const areAnyUnselectableChildrenUnexpanded = !!parentChildren.find(
+        (parent) => !selectableChildren.find((child) => child.parentValue === parent.value),
+    );
+
+    const isSelected = useMemo(
+        () =>
+            !!selectedOptions.find((o) => o.value === option.value) ||
+            (!areParentsSelectable &&
+                !!option.isParent &&
+                !!selectableChildren.length &&
+                areAllChildrenSelected &&
+                !areAnyUnselectableChildrenUnexpanded),
+        [
+            selectedOptions,
+            areAllChildrenSelected,
+            areAnyUnselectableChildrenUnexpanded,
+            areParentsSelectable,
+            option.isParent,
+            option.value,
+            selectableChildren.length,
+        ],
+    );
+
+    const isImplicitlySelected = useMemo(
+        () => !option.isParent && !!selectedOptions.find((o) => o.value === option.parentValue),
+        [selectedOptions, option.isParent, option.parentValue],
+    );
+
+    const isParentMissingChildren = useMemo(() => !!option.isParent && !children.length, [children, option.isParent]);
+
+    const isPartialSelected = useMemo(
+        () =>
+            (!areAllChildrenSelected && areAnyChildrenSelected) ||
+            (isSelected && isParentMissingChildren) ||
+            (isSelected && areAnyUnselectableChildrenUnexpanded) ||
+            (areAnyUnselectableChildrenUnexpanded && areAnyChildrenSelected) ||
+            (isSelected && !!children.length && !areAnyChildrenSelected),
+        [
+            isSelected,
+            children,
+            areAllChildrenSelected,
+            areAnyChildrenSelected,
+            areAnyUnselectableChildrenUnexpanded,
+            isParentMissingChildren,
+        ],
+    );
+
+    const selectOption = () => {
+        if (areParentsSelectable && option.isParent) {
+            const existingSelectedOptions = new Set(selectedOptions.map((opt) => opt.value));
+            const existingChildSelectedOptions =
+                selectedOptions.filter((opt) => opt.parentValue === option.value) || [];
+            if (existingSelectedOptions.has(option.value)) {
+                removeOptions([option]);
+            } else {
+                // filter out the childrens of parent selection as we are allowing implicitly selection
+                const filteredOptions = selectedOptions.filter(
+                    (selectedOption) => !existingChildSelectedOptions.find((o) => o.value === selectedOption.value),
+                );
+                const newSelectedOptions = [...filteredOptions, option];
+
+                setSelectedOptions(newSelectedOptions);
+            }
+        } else if (isPartialSelected || (!isSelected && !areAnyChildrenSelected)) {
+            const optionsToAdd = option.isParent && !areParentsSelectable ? selectableChildren : [option];
+            addOptions(optionsToAdd);
+        } else if (areAllChildrenSelected) {
+            removeOptions([option, ...selectableChildren]);
+        } else {
+            handleOptionChange(option);
+        }
+    };
+
+    // one loader variable for fetching data for expanded parents and their respective child nodes
+    useEffect(() => {
+        // once loading has been done just remove all the parent node urn
+        if (!isLoadingParentChildList) {
+            setLoadingParentUrns([]);
+        }
+    }, [isLoadingParentChildList]);
+
+    return (
+        <div>
+            <ParentOption>
+                <OptionLabel
+                    key={option.value}
+                    onClick={(e) => {
+                        e.preventDefault();
+                        if (isImplicitlySelected) {
+                            return;
+                        }
+                        if (isParentMissingChildren) {
+                            setLoadingParentUrns((previousIds) => [...previousIds, option.value]);
+                            loadData?.(option);
+                        }
+                        if (option.isParent) {
+                            setIsOpen(!isOpen);
+                        } else {
+                            selectOption();
+                        }
+                    }}
+                    isSelected={!isMultiSelect && isSelected}
+                    // added hack to show cursor in wait untill we get the inline spinner
+                    style={{ width: '100%', cursor: loadingParentUrns.includes(option.value) ? 'wait' : 'pointer' }}
+                >
+                    {option.isParent && <strong>{option.label}</strong>}
+                    {!option.isParent && <>{option.label}</>}
+                    {option.isParent && (
+                        <Icon
+                            onClick={(e) => {
+                                e.stopPropagation();
+                                e.preventDefault();
+                                setIsOpen(!isOpen);
+                                if (!isOpen && isParentMissingChildren) {
+                                    setLoadingParentUrns((previousIds) => [...previousIds, option.value]);
+                                    loadData?.(option);
+                                }
+                            }}
+                            icon="ChevronLeft"
+                            rotate={isOpen ? '90' : '270'}
+                            size="xl"
+                            color="gray"
+                            style={{ cursor: 'pointer', marginLeft: '4px' }}
+                        />
+                    )}
+                    <StyledCheckbox
+                        checked={isImplicitlySelected || isSelected}
+                        indeterminate={
+                            areParentsSelectable && option.isParent ? areAnyChildrenSelected : isPartialSelected
+                        }
+                        onClick={(e) => {
+                            e.preventDefault();
+                            if (isImplicitlySelected) {
+                                return;
+                            }
+                            e.stopPropagation();
+                            if (isParentMissingChildren) {
+                                loadData?.(option);
+                                if (!areParentsSelectable) {
+                                    setAutoSelectChildren(true);
+                                }
+                            }
+                            selectOption();
+                        }}
+                        disabled={isImplicitlySelected}
+                    />
+                </OptionLabel>
+            </ParentOption>
+            {isOpen && (
+                <ChildOptions>
+                    {directChildren.map((child) => (
+                        <NestedOption
+                            key={child.value}
+                            selectedOptions={selectedOptions}
+                            option={child}
+                            parentValueToOptions={parentValueToOptions}
+                            addOptions={addOptions}
+                            handleOptionChange={handleOptionChange}
+                            loadData={loadData}
+                            removeOptions={removeOptions}
+                            isMultiSelect={isMultiSelect}
+                            areParentsSelectable={areParentsSelectable}
+                            setSelectedOptions={setSelectedOptions}
+                        />
+                    ))}
+                </ChildOptions>
+            )}
+        </div>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedSelect.tsx b/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedSelect.tsx
new file mode 100644
index 0000000000000..744c7bfcfec0d
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedSelect.tsx
@@ -0,0 +1,312 @@
+import React, { useCallback, useEffect, useRef, useState } from 'react';
+import styled from 'styled-components';
+
+import { Icon, Pill } from '@components';
+
+import {
+    ActionButtonsContainer,
+    Container,
+    Dropdown,
+    OptionList,
+    Placeholder,
+    SearchIcon,
+    SearchInput,
+    SearchInputContainer,
+    SelectBase,
+    SelectLabel,
+    StyledClearButton,
+} from '../components';
+
+import { SelectSizeOptions } from '../types';
+import { NestedOption } from './NestedOption';
+import { SelectOption } from './types';
+
+const NO_PARENT_VALUE = 'no_parent_value';
+
+const LabelDisplayWrapper = styled.div`
+    display: flex;
+    flex-wrap: wrap;
+    gap: 4px;
+    max-height: 125px;
+    min-height: 16px;
+`;
+
+interface SelectLabelDisplayProps {
+    selectedOptions: SelectOption[];
+    placeholder: string;
+    handleOptionChange: (node: SelectOption) => void;
+}
+
+const SelectLabelDisplay = ({ selectedOptions, placeholder, handleOptionChange }: SelectLabelDisplayProps) => {
+    return (
+        <LabelDisplayWrapper>
+            {!!selectedOptions.length &&
+                selectedOptions.map((o) => (
+                    <Pill
+                        label={o.label}
+                        rightIcon="Close"
+                        size="sm"
+                        onClickRightIcon={(e) => {
+                            e.stopPropagation();
+                            handleOptionChange(o);
+                        }}
+                    />
+                ))}
+            {!selectedOptions.length && <Placeholder>{placeholder}</Placeholder>}
+        </LabelDisplayWrapper>
+    );
+};
+
+export interface ActionButtonsProps {
+    fontSize?: SelectSizeOptions;
+    selectedOptions: SelectOption[];
+    isOpen: boolean;
+    isDisabled: boolean;
+    isReadOnly: boolean;
+    handleClearSelection: () => void;
+}
+
+const SelectActionButtons = ({
+    selectedOptions,
+    isOpen,
+    isDisabled,
+    isReadOnly,
+    handleClearSelection,
+    fontSize = 'md',
+}: ActionButtonsProps) => {
+    return (
+        <ActionButtonsContainer>
+            {!!selectedOptions.length && !isDisabled && !isReadOnly && (
+                <StyledClearButton
+                    icon="Close"
+                    isCircle
+                    onClick={handleClearSelection}
+                    size={fontSize}
+                    data-testid="dropdown-option-clear-icon"
+                />
+            )}
+            <Icon icon="ChevronLeft" rotate={isOpen ? '90' : '270'} size="xl" color="gray" />
+        </ActionButtonsContainer>
+    );
+};
+
+export interface SelectProps {
+    options: SelectOption[];
+    label: string;
+    value?: string;
+    initialValues?: SelectOption[];
+    onCancel?: () => void;
+    onUpdate?: (selectedValues: SelectOption[]) => void;
+    size?: SelectSizeOptions;
+    showSearch?: boolean;
+    isDisabled?: boolean;
+    isReadOnly?: boolean;
+    isRequired?: boolean;
+    isMultiSelect?: boolean;
+    areParentsSelectable?: boolean;
+    loadData?: (node: SelectOption) => void;
+    onSearch?: (query: string) => void;
+    width?: number | 'full';
+    height?: number;
+    placeholder?: string;
+    searchPlaceholder?: string;
+    isLoadingParentChildList?: boolean;
+}
+
+export const selectDefaults: SelectProps = {
+    options: [],
+    label: '',
+    size: 'md',
+    showSearch: false,
+    isDisabled: false,
+    isReadOnly: false,
+    isRequired: false,
+    isMultiSelect: false,
+    width: 255,
+    height: 425,
+};
+
+export const NestedSelect = ({
+    options = selectDefaults.options,
+    label = selectDefaults.label,
+    initialValues = [],
+    onUpdate,
+    loadData,
+    onSearch,
+    showSearch = selectDefaults.showSearch,
+    isDisabled = selectDefaults.isDisabled,
+    isReadOnly = selectDefaults.isReadOnly,
+    isRequired = selectDefaults.isRequired,
+    isMultiSelect = selectDefaults.isMultiSelect,
+    size = selectDefaults.size,
+    areParentsSelectable = true,
+    placeholder,
+    searchPlaceholder,
+    height = selectDefaults.height,
+    isLoadingParentChildList = false,
+    ...props
+}: SelectProps) => {
+    const [searchQuery, setSearchQuery] = useState('');
+    const [isOpen, setIsOpen] = useState(false);
+    const [selectedOptions, setSelectedOptions] = useState<SelectOption[]>(initialValues);
+    const selectRef = useRef<HTMLDivElement>(null);
+
+    // TODO: handle searching inside of a nested component on the FE only
+
+    const handleDocumentClick = useCallback((e: MouseEvent) => {
+        if (selectRef.current && !selectRef.current.contains(e.target as Node)) {
+            setIsOpen(false);
+        }
+    }, []);
+
+    useEffect(() => {
+        document.addEventListener('click', handleDocumentClick);
+        return () => {
+            document.removeEventListener('click', handleDocumentClick);
+        };
+    }, [handleDocumentClick]);
+
+    const handleSelectClick = useCallback(() => {
+        if (!isDisabled && !isReadOnly) {
+            setIsOpen((prev) => !prev);
+        }
+    }, [isDisabled, isReadOnly]);
+
+    const handleSearch = useCallback(
+        (query: string) => {
+            setSearchQuery(query);
+            onSearch?.(query);
+        },
+        [onSearch],
+    );
+
+    // Instead of calling the update function individually whenever selectedOptions changes,
+    // we use the useEffect hook to trigger the onUpdate function automatically when selectedOptions is updated.
+    useEffect(() => {
+        if (onUpdate) {
+            onUpdate(selectedOptions);
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [selectedOptions]);
+
+    const handleOptionChange = useCallback(
+        (option: SelectOption) => {
+            let newSelectedOptions: SelectOption[];
+            if (selectedOptions.find((o) => o.value === option.value)) {
+                newSelectedOptions = selectedOptions.filter((o) => o.value !== option.value);
+            } else {
+                newSelectedOptions = [...selectedOptions, option];
+            }
+            setSelectedOptions(newSelectedOptions);
+            if (!isMultiSelect) {
+                setIsOpen(false);
+            }
+        },
+        [selectedOptions, isMultiSelect],
+    );
+
+    const addOptions = useCallback(
+        (optionsToAdd: SelectOption[]) => {
+            const existingValues = new Set(selectedOptions.map((option) => option.value));
+            const filteredOptionsToAdd = optionsToAdd.filter((option) => !existingValues.has(option.value));
+            if (filteredOptionsToAdd.length) {
+                const newSelectedOptions = [...selectedOptions, ...filteredOptionsToAdd];
+                setSelectedOptions(newSelectedOptions);
+            }
+        },
+        [selectedOptions],
+    );
+
+    const removeOptions = useCallback(
+        (optionsToRemove: SelectOption[]) => {
+            const newValues = selectedOptions.filter(
+                (selectedOption) => !optionsToRemove.find((o) => o.value === selectedOption.value),
+            );
+            setSelectedOptions(newValues);
+        },
+        [selectedOptions],
+    );
+
+    const handleClearSelection = useCallback(() => {
+        setSelectedOptions([]);
+        setIsOpen(false);
+        if (onUpdate) {
+            onUpdate([]);
+        }
+    }, [onUpdate]);
+
+    // generate map for options to quickly fetch children
+    const parentValueToOptions: { [parentValue: string]: SelectOption[] } = {};
+    options.forEach((o) => {
+        const parentValue = o.parentValue || NO_PARENT_VALUE;
+        parentValueToOptions[parentValue] = parentValueToOptions[parentValue]
+            ? [...parentValueToOptions[parentValue], o]
+            : [o];
+    });
+
+    const rootOptions = parentValueToOptions[NO_PARENT_VALUE] || [];
+
+    return (
+        <Container ref={selectRef} size={size || 'md'} width={props.width || 255}>
+            {label && <SelectLabel onClick={handleSelectClick}>{label}</SelectLabel>}
+            <SelectBase
+                isDisabled={isDisabled}
+                isReadOnly={isReadOnly}
+                isRequired={isRequired}
+                isOpen={isOpen}
+                onClick={handleSelectClick}
+                fontSize={size}
+                data-testid="nested-options-dropdown-container"
+                {...props}
+            >
+                <SelectLabelDisplay
+                    selectedOptions={selectedOptions}
+                    placeholder={placeholder || 'Select an option'}
+                    handleOptionChange={handleOptionChange}
+                />
+                <SelectActionButtons
+                    selectedOptions={selectedOptions}
+                    isOpen={isOpen}
+                    isDisabled={!!isDisabled}
+                    isReadOnly={!!isReadOnly}
+                    handleClearSelection={handleClearSelection}
+                    fontSize={size}
+                />
+            </SelectBase>
+            {isOpen && (
+                <Dropdown style={{ maxHeight: height, overflow: 'auto' }}>
+                    {showSearch && (
+                        <SearchInputContainer>
+                            <SearchInput
+                                type="text"
+                                placeholder={searchPlaceholder || 'Search...'}
+                                value={searchQuery}
+                                onChange={(e) => handleSearch(e.target.value)}
+                                style={{ fontSize: size || 'md', width: '100%' }}
+                            />
+                            <SearchIcon icon="Search" size={size} color="gray" />
+                        </SearchInputContainer>
+                    )}
+                    <OptionList>
+                        {rootOptions.map((option) => (
+                            <NestedOption
+                                key={option.value}
+                                selectedOptions={selectedOptions}
+                                option={option}
+                                parentValueToOptions={parentValueToOptions}
+                                handleOptionChange={handleOptionChange}
+                                addOptions={addOptions}
+                                removeOptions={removeOptions}
+                                loadData={loadData}
+                                isMultiSelect={isMultiSelect}
+                                setSelectedOptions={setSelectedOptions}
+                                areParentsSelectable={areParentsSelectable}
+                                isLoadingParentChildList={isLoadingParentChildList}
+                            />
+                        ))}
+                    </OptionList>
+                </Dropdown>
+            )}
+        </Container>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Select/Nested/types.ts b/datahub-web-react/src/alchemy-components/components/Select/Nested/types.ts
new file mode 100644
index 0000000000000..62d4541fce0d3
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/Nested/types.ts
@@ -0,0 +1,9 @@
+import { Entity } from '@src/types.generated';
+
+export interface SelectOption {
+    value: string;
+    label: string;
+    parentValue?: string;
+    isParent?: boolean;
+    entity?: Entity;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Select/Select.stories.tsx b/datahub-web-react/src/alchemy-components/components/Select/Select.stories.tsx
new file mode 100644
index 0000000000000..0ec20b15e771a
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/Select.stories.tsx
@@ -0,0 +1,431 @@
+import { GridList } from '@components/.docs/mdx-components';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import type { Meta, StoryObj } from '@storybook/react';
+import React from 'react';
+import { Select, selectDefaults } from './Select';
+import { SimpleSelect } from './SimpleSelect';
+import { SelectSizeOptions } from './types';
+
+// Auto Docs
+const meta: Meta = {
+    title: 'Forms / Select',
+    component: Select,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'This component allows users to select one or multiple input options from a dropdown list.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        options: {
+            description: 'Array of options for the Select component.',
+            control: {
+                type: 'object',
+            },
+            table: {
+                defaultValue: { summary: JSON.stringify(selectDefaults.options) },
+            },
+        },
+        label: {
+            description: 'Label for the Select component.',
+            control: {
+                type: 'text',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.label },
+            },
+        },
+        values: {
+            description: 'Selected values for the Select component.',
+            control: {
+                type: 'object',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.values?.toString() },
+            },
+        },
+        showSearch: {
+            description: 'Whether to show the search input.',
+            control: {
+                type: 'boolean',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.showSearch?.toString() },
+            },
+        },
+        isDisabled: {
+            description: 'Whether the Select component is disabled.',
+            control: {
+                type: 'boolean',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.isDisabled?.toString() },
+            },
+        },
+        isReadOnly: {
+            description: 'Whether the Select component is read-only.',
+            control: {
+                type: 'boolean',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.isReadOnly?.toString() },
+            },
+        },
+        isRequired: {
+            description: 'Whether the Select component is required.',
+            control: {
+                type: 'boolean',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.isRequired?.toString() },
+            },
+        },
+        size: {
+            description: 'Size of the Select component.',
+            control: {
+                type: 'select',
+                options: ['sm', 'md', 'lg'],
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.size },
+            },
+        },
+        width: {
+            description: 'Width of the Select component.',
+            control: {
+                type: 'number',
+            },
+            table: {
+                defaultValue: { summary: `${selectDefaults.width}` },
+            },
+        },
+        isMultiSelect: {
+            description: 'Whether the Select component allows multiple values to be selected.',
+            control: {
+                type: 'boolean',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.isMultiSelect?.toString() },
+            },
+        },
+        placeholder: {
+            description: 'Placeholder for the Select component.',
+            control: {
+                type: 'text',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.placeholder },
+            },
+        },
+        disabledValues: {
+            description: 'Disabled values for the multi-select component.',
+            control: {
+                type: 'object',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.disabledValues?.toString() },
+            },
+        },
+        showSelectAll: {
+            description: 'Whether the multi select component shows Select All button.',
+            control: {
+                type: 'boolean',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.showSelectAll?.toString() },
+            },
+        },
+        selectAllLabel: {
+            description: 'Label for Select All button.',
+            control: {
+                type: 'text',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.selectAllLabel },
+            },
+        },
+        showDescriptions: {
+            description: 'Whether to show descriptions with the select options.',
+            control: {
+                type: 'boolean',
+            },
+            table: {
+                defaultValue: { summary: selectDefaults.showDescriptions?.toString() },
+            },
+        },
+    },
+
+    // Define defaults
+    args: {
+        options: [
+            { label: 'Option 1', value: '1' },
+            { label: 'Option 2', value: '2' },
+            { label: 'Option 3', value: '3' },
+        ],
+        label: 'Select Label',
+        values: undefined,
+        showSearch: selectDefaults.showSearch,
+        isDisabled: selectDefaults.isDisabled,
+        isReadOnly: selectDefaults.isReadOnly,
+        isRequired: selectDefaults.isRequired,
+        onCancel: () => console.log('Cancel clicked'),
+        onUpdate: (selectedValues: string[]) => console.log('Update clicked', selectedValues),
+        size: 'md', // Default size
+        width: 255,
+        isMultiSelect: selectDefaults.isMultiSelect,
+        placeholder: selectDefaults.placeholder,
+        disabledValues: undefined,
+        showSelectAll: false,
+        selectAllLabel: 'Select All',
+        showDescriptions: false,
+    },
+} satisfies Meta<typeof Select>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+const sizeOptions: SelectSizeOptions[] = ['sm', 'md', 'lg'];
+
+export const simpleSelectSandbox: Story = {
+    tags: ['dev'],
+
+    render: (props) => (
+        <SimpleSelect
+            options={props.options}
+            label={props.label}
+            values={props.values}
+            showSearch={props.showSearch}
+            isDisabled={props.isDisabled}
+            isReadOnly={props.isReadOnly}
+            isRequired={props.isRequired}
+            size={props.size}
+            width={props.width}
+            onUpdate={props.onUpdate} // Optional: to log the selected value
+            isMultiSelect={props.isMultiSelect}
+            placeholder={props.placeholder}
+            disabledValues={props.disabledValues}
+        />
+    ),
+};
+
+export const simpleSelectStates = () => (
+    <GridList isVertical>
+        <>
+            <SimpleSelect options={[{ label: 'Default', value: 'default' }]} label="Default" values={['default']} />
+            <SimpleSelect
+                options={[{ label: 'Disabled State', value: 'disabled' }]}
+                label="Disabled State"
+                isDisabled
+                values={['disabled']}
+            />
+            <SimpleSelect
+                options={[{ label: 'Read Only State', value: 'readonly' }]}
+                label="Read Only State"
+                isReadOnly
+                values={['readonly']}
+            />
+        </>
+    </GridList>
+);
+
+export const simpleSelectWithSearch = () => (
+    <SimpleSelect
+        options={[
+            { label: 'Option 1', value: '1' },
+            { label: 'Option 2', value: '2' },
+            { label: 'Option 3', value: '3' },
+        ]}
+        label="Simple Select with Search"
+        showSearch
+        values={['2']}
+    />
+);
+
+export const simpleSelectWithMultiSelect = () => (
+    <SimpleSelect
+        options={[
+            { label: 'Option 1', value: '1' },
+            { label: 'Option 2', value: '2' },
+            { label: 'Option 3', value: '3' },
+        ]}
+        label="Simple Select with multi-select"
+        showSearch
+        values={['2', '3']}
+        isMultiSelect
+    />
+);
+
+export const simpleSelectWithDisabledValues = () => (
+    <SimpleSelect
+        options={[
+            { label: 'Option 1', value: '1' },
+            { label: 'Option 2', value: '2' },
+            { label: 'Option 3', value: '3' },
+        ]}
+        label="Simple Select with disabled values"
+        values={['2']}
+        isMultiSelect
+        disabledValues={['2']}
+    />
+);
+
+export const simpleSelectWithSelectAll = () => (
+    <SimpleSelect
+        options={[
+            { label: 'Option 1', value: '1' },
+            { label: 'Option 2', value: '2' },
+            { label: 'Option 3', value: '3' },
+        ]}
+        label="Simple Select with Select All"
+        isMultiSelect
+        showSelectAll
+    />
+);
+
+export const simpleSelectWithDescriptions = () => (
+    <SimpleSelect
+        options={[
+            { label: 'Option 1', value: '1', description: 'Description of option 1' },
+            { label: 'Option 2', value: '2', description: 'Description of option 2 is  longgggggg' },
+            { label: 'Option 3', value: '3', description: 'Description of option 3' },
+        ]}
+        label="Simple Select with descriptions"
+        showDescriptions
+    />
+);
+
+export const simpleSelectSizes = () => (
+    <GridList isVertical>
+        {sizeOptions.map((size, index) => (
+            <SimpleSelect
+                key={`simpleselect-${size}`}
+                options={[
+                    { label: 'Option 1', value: '1' },
+                    { label: 'Option 2', value: '2' },
+                    { label: 'Option 3', value: '3' },
+                ]}
+                label={`Simple Select - Font Size: ${size}, Width: ${255 + 50 * index}px`}
+                values={['3']}
+                size={size}
+                width={255 + 50 * index}
+            />
+        ))}
+    </GridList>
+);
+
+// Basic story is what is displayed 1st in storybook & is used as the code sandbox
+// Pass props to this so that it can be customized via the UI props panel
+export const BasicSelectSandbox: Story = {
+    tags: ['dev'],
+
+    render: (props) => (
+        <Select
+            options={props.options}
+            label={props.label}
+            values={props.values}
+            onCancel={props.onCancel}
+            onUpdate={props.onUpdate}
+            showSearch={props.showSearch}
+            isDisabled={props.isDisabled}
+            isReadOnly={props.isReadOnly}
+            isRequired={props.isRequired}
+            size={props.size}
+            width={props.width}
+            isMultiSelect={props.isMultiSelect}
+            placeholder={props.placeholder}
+        />
+    ),
+};
+
+export const states = () => (
+    <GridList isVertical>
+        <>
+            <Select options={[{ label: 'Default', value: 'default' }]} label="Default" values={['default']} />
+            <Select
+                options={[{ label: 'Disabled State', value: 'disabled' }]}
+                label="Disabled State"
+                isDisabled
+                values={['disabled']}
+            />
+            <Select
+                options={[{ label: 'Read Only State', value: 'readonly' }]}
+                label="Read Only State"
+                isReadOnly
+                values={['readonly']}
+            />
+        </>
+    </GridList>
+);
+
+export const withSearch = () => (
+    <Select
+        options={[
+            { label: 'Option 1', value: '1' },
+            { label: 'Option 2', value: '2' },
+            { label: 'Option 3', value: '3' },
+        ]}
+        label="Select with Search"
+        showSearch
+        values={['2']}
+    />
+);
+
+export const withMultiSelect = () => (
+    <Select
+        options={[
+            { label: 'Option 1', value: '1' },
+            { label: 'Option 2', value: '2' },
+            { label: 'Option 3', value: '3' },
+        ]}
+        label="Select with multi-select"
+        showSearch
+        values={['2', '3']}
+        isMultiSelect
+    />
+);
+
+export const sizes = () => (
+    <GridList isVertical>
+        {sizeOptions.map((size, index) => (
+            <Select
+                key={`select-${size}`}
+                options={[
+                    { label: 'Option 1', value: '1' },
+                    { label: 'Option 2', value: '2' },
+                    { label: 'Option 3', value: '3' },
+                ]}
+                label={`Select - Font Size: ${size}, Width: ${255 + 50 * index}px`}
+                values={['3']}
+                onCancel={() => alert('Cancel clicked')}
+                onUpdate={(selectedValues) => alert(`Update clicked with values: ${selectedValues}`)}
+                size={size}
+                width={255 + 50 * index}
+            />
+        ))}
+    </GridList>
+);
+
+export const footerActions = () => (
+    <GridList isVertical>
+        <Select
+            options={[
+                { label: 'Option 1', value: '1' },
+                { label: 'Option 2', value: '2' },
+                { label: 'Option 3', value: '3' },
+            ]}
+            label="Select with Footer Actions"
+            values={['3']}
+            onCancel={() => alert('Cancel clicked')}
+            onUpdate={(selectedValues) => alert(`Update clicked with values: ${selectedValues}`)}
+            size="md"
+        />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Select/Select.tsx b/datahub-web-react/src/alchemy-components/components/Select/Select.tsx
new file mode 100644
index 0000000000000..da28f09056543
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/Select.tsx
@@ -0,0 +1,65 @@
+import React from 'react';
+import { BasicSelect } from './BasicSelect';
+import { SelectProps } from './types';
+
+export const selectDefaults: SelectProps = {
+    options: [],
+    label: '',
+    showSearch: false,
+    values: undefined,
+    size: 'md',
+    isDisabled: false,
+    isReadOnly: false,
+    isRequired: false,
+    width: 255,
+    isMultiSelect: false,
+    placeholder: 'Select an option',
+    disabledValues: undefined,
+    showSelectAll: false,
+    selectAllLabel: 'Select All',
+    showDescriptions: false,
+};
+
+export const Select = ({
+    options = selectDefaults.options,
+    label = selectDefaults.label,
+    values = [],
+    onCancel,
+    onUpdate,
+    showSearch = selectDefaults.showSearch,
+    isDisabled = selectDefaults.isDisabled,
+    isReadOnly = selectDefaults.isReadOnly,
+    isRequired = selectDefaults.isRequired,
+    size = selectDefaults.size,
+    width = selectDefaults.width,
+    isMultiSelect = selectDefaults.isMultiSelect,
+    placeholder = selectDefaults.placeholder,
+    disabledValues = selectDefaults.disabledValues,
+    showSelectAll = selectDefaults.showSelectAll,
+    selectAllLabel = selectDefaults.selectAllLabel,
+    showDescriptions = selectDefaults.showDescriptions,
+    ...props
+}: SelectProps) => {
+    return (
+        <BasicSelect
+            options={options}
+            size={size}
+            label={label}
+            values={values}
+            showSearch={showSearch}
+            isDisabled={isDisabled}
+            isReadOnly={isReadOnly}
+            isRequired={isRequired}
+            onCancel={onCancel}
+            onUpdate={onUpdate}
+            width={width}
+            isMultiSelect={isMultiSelect}
+            placeholder={placeholder}
+            disabledValues={disabledValues}
+            showSelectAll={showSelectAll}
+            selectAllLabel={selectAllLabel}
+            showDescriptions={showDescriptions}
+            {...props}
+        />
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Select/SimpleSelect.tsx b/datahub-web-react/src/alchemy-components/components/Select/SimpleSelect.tsx
new file mode 100644
index 0000000000000..be1184cee9e9f
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/SimpleSelect.tsx
@@ -0,0 +1,299 @@
+import { Icon, Pill, Text } from '@components';
+import { isEqual } from 'lodash';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import {
+    ActionButtonsContainer,
+    Container,
+    DescriptionContainer,
+    Dropdown,
+    LabelContainer,
+    LabelsWrapper,
+    OptionContainer,
+    OptionLabel,
+    OptionList,
+    Placeholder,
+    SearchIcon,
+    SearchInput,
+    SearchInputContainer,
+    SelectAllOption,
+    SelectBase,
+    SelectLabel,
+    SelectValue,
+    StyledCheckbox,
+    StyledClearButton,
+} from './components';
+import { ActionButtonsProps, SelectLabelDisplayProps, SelectOption, SelectProps } from './types';
+
+const SelectLabelDisplay = ({
+    selectedValues,
+    options,
+    placeholder,
+    isMultiSelect,
+    removeOption,
+    disabledValues,
+    showDescriptions,
+}: SelectLabelDisplayProps) => {
+    const selectedOptions = options.filter((opt) => selectedValues.includes(opt.value));
+    return (
+        <LabelsWrapper>
+            {!!selectedOptions.length &&
+                isMultiSelect &&
+                selectedOptions.map((o) => {
+                    const isDisabled = disabledValues?.includes(o.value);
+                    return (
+                        <Pill
+                            label={o.label}
+                            rightIcon={!isDisabled ? 'Close' : ''}
+                            size="sm"
+                            key={o.value}
+                            onClickRightIcon={(e) => {
+                                e.stopPropagation();
+                                removeOption?.(o);
+                            }}
+                            clickable={!isDisabled}
+                        />
+                    );
+                })}
+            {!selectedValues.length && <Placeholder>{placeholder}</Placeholder>}
+            {!isMultiSelect && (
+                <>
+                    <SelectValue>{selectedOptions[0]?.label}</SelectValue>
+                    {showDescriptions && !!selectedValues.length && (
+                        <DescriptionContainer>{selectedOptions[0]?.description}</DescriptionContainer>
+                    )}
+                </>
+            )}
+        </LabelsWrapper>
+    );
+};
+
+const SelectActionButtons = ({
+    selectedValues,
+    isOpen,
+    isDisabled,
+    isReadOnly,
+    showClear,
+    handleClearSelection,
+    fontSize = 'md',
+}: ActionButtonsProps) => {
+    return (
+        <ActionButtonsContainer>
+            {showClear && selectedValues.length > 0 && !isDisabled && !isReadOnly && (
+                <StyledClearButton icon="Close" isCircle onClick={handleClearSelection} size={fontSize} />
+            )}
+            <Icon icon="ChevronLeft" rotate={isOpen ? '90' : '270'} size="xl" color="gray" />
+        </ActionButtonsContainer>
+    );
+};
+
+export const selectDefaults: SelectProps = {
+    options: [],
+    label: '',
+    size: 'md',
+    showSearch: false,
+    isDisabled: false,
+    isReadOnly: false,
+    isRequired: false,
+    showClear: true,
+    width: 255,
+    isMultiSelect: false,
+    placeholder: 'Select an option ',
+    showSelectAll: false,
+    selectAllLabel: 'Select All',
+    showDescriptions: false,
+};
+
+export const SimpleSelect = ({
+    options = selectDefaults.options,
+    label = selectDefaults.label,
+    values = [],
+    onUpdate,
+    showSearch = selectDefaults.showSearch,
+    isDisabled = selectDefaults.isDisabled,
+    isReadOnly = selectDefaults.isReadOnly,
+    isRequired = selectDefaults.isRequired,
+    showClear = selectDefaults.showClear,
+    size = selectDefaults.size,
+    isMultiSelect = selectDefaults.isMultiSelect,
+    placeholder = selectDefaults.placeholder,
+    disabledValues = [],
+    showSelectAll = selectDefaults.showSelectAll,
+    selectAllLabel = selectDefaults.selectAllLabel,
+    optionListTestId,
+    showDescriptions = selectDefaults.showDescriptions,
+    ...props
+}: SelectProps) => {
+    const [searchQuery, setSearchQuery] = useState('');
+    const [isOpen, setIsOpen] = useState(false);
+    const [selectedValues, setSelectedValues] = useState<string[]>(values);
+    const selectRef = useRef<HTMLDivElement>(null);
+    const [areAllSelected, setAreAllSelected] = useState(false);
+
+    useEffect(() => {
+        if (values?.length > 0 && !isEqual(selectedValues, values)) {
+            setSelectedValues(values);
+        }
+    }, [values, selectedValues]);
+
+    useEffect(() => {
+        setAreAllSelected(selectedValues.length === options.length);
+    }, [options, selectedValues]);
+
+    const filteredOptions = useMemo(
+        () => options.filter((option) => option.label.toLowerCase().includes(searchQuery.toLowerCase())),
+        [options, searchQuery],
+    );
+
+    const handleDocumentClick = useCallback((e: MouseEvent) => {
+        if (selectRef.current && !selectRef.current.contains(e.target as Node)) {
+            setIsOpen(false);
+        }
+    }, []);
+
+    useEffect(() => {
+        document.addEventListener('click', handleDocumentClick);
+        return () => {
+            document.removeEventListener('click', handleDocumentClick);
+        };
+    }, [handleDocumentClick]);
+
+    const handleSelectClick = useCallback(() => {
+        if (!isDisabled && !isReadOnly) {
+            setIsOpen((prev) => !prev);
+        }
+    }, [isDisabled, isReadOnly]);
+
+    const handleOptionChange = useCallback(
+        (option: SelectOption) => {
+            const updatedValues = selectedValues.includes(option.value)
+                ? selectedValues.filter((val) => val !== option.value)
+                : [...selectedValues, option.value];
+
+            setSelectedValues(isMultiSelect ? updatedValues : [option.value]);
+            if (onUpdate) {
+                onUpdate(isMultiSelect ? updatedValues : [option.value]);
+            }
+            if (!isMultiSelect) setIsOpen(false);
+        },
+        [onUpdate, isMultiSelect, selectedValues],
+    );
+
+    const handleClearSelection = useCallback(() => {
+        setSelectedValues([]);
+        setAreAllSelected(false);
+        setIsOpen(false);
+        if (onUpdate) {
+            onUpdate([]);
+        }
+    }, [onUpdate]);
+
+    const handleSelectAll = () => {
+        if (areAllSelected) {
+            setSelectedValues([]);
+            onUpdate?.([]);
+        } else {
+            const allValues = options.map((option) => option.value);
+            setSelectedValues(allValues);
+            onUpdate?.(allValues);
+        }
+        setAreAllSelected(!areAllSelected);
+    };
+
+    return (
+        <Container ref={selectRef} size={size || 'md'} width={props.width || 255}>
+            {label && <SelectLabel onClick={handleSelectClick}>{label}</SelectLabel>}
+            <SelectBase
+                isDisabled={isDisabled}
+                isReadOnly={isReadOnly}
+                isRequired={isRequired}
+                isOpen={isOpen}
+                onClick={handleSelectClick}
+                fontSize={size}
+                {...props}
+            >
+                <SelectLabelDisplay
+                    selectedValues={selectedValues}
+                    options={options}
+                    placeholder={placeholder || 'Select an option'}
+                    isMultiSelect={isMultiSelect}
+                    removeOption={handleOptionChange}
+                    disabledValues={disabledValues}
+                    showDescriptions={showDescriptions}
+                />
+                <SelectActionButtons
+                    selectedValues={selectedValues}
+                    isOpen={isOpen}
+                    isDisabled={!!isDisabled}
+                    isReadOnly={!!isReadOnly}
+                    handleClearSelection={handleClearSelection}
+                    fontSize={size}
+                    showClear={!!showClear}
+                />
+            </SelectBase>
+            {isOpen && (
+                <Dropdown>
+                    {showSearch && (
+                        <SearchInputContainer>
+                            <SearchInput
+                                type="text"
+                                placeholder="Search…"
+                                value={searchQuery}
+                                onChange={(e) => setSearchQuery(e.target.value)}
+                                style={{ fontSize: size || 'md' }}
+                            />
+                            <SearchIcon icon="Search" size={size} color="gray" />
+                        </SearchInputContainer>
+                    )}
+                    <OptionList data-testid={optionListTestId}>
+                        {showSelectAll && isMultiSelect && (
+                            <SelectAllOption
+                                isSelected={areAllSelected}
+                                onClick={() => !(disabledValues.length === options.length) && handleSelectAll()}
+                                isDisabled={disabledValues.length === options.length}
+                            >
+                                <LabelContainer>
+                                    <span>{selectAllLabel}</span>
+                                    <StyledCheckbox
+                                        checked={areAllSelected}
+                                        disabled={disabledValues.length === options.length}
+                                    />
+                                </LabelContainer>
+                            </SelectAllOption>
+                        )}
+                        {filteredOptions.map((option) => (
+                            <OptionLabel
+                                key={option.value}
+                                onClick={() => !isMultiSelect && handleOptionChange(option)}
+                                isSelected={selectedValues.includes(option.value)}
+                                isMultiSelect={isMultiSelect}
+                                isDisabled={disabledValues?.includes(option.value)}
+                            >
+                                {isMultiSelect ? (
+                                    <LabelContainer>
+                                        <span>{option.label}</span>
+                                        <StyledCheckbox
+                                            onClick={() => handleOptionChange(option)}
+                                            checked={selectedValues.includes(option.value)}
+                                            disabled={disabledValues?.includes(option.value)}
+                                        />
+                                    </LabelContainer>
+                                ) : (
+                                    <OptionContainer>
+                                        <Text color="gray" weight="semiBold" size="md">
+                                            {option.label}
+                                        </Text>
+                                        {!!option.description && (
+                                            <Text color="gray" weight="normal" size="sm">
+                                                {option.description}
+                                            </Text>
+                                        )}
+                                    </OptionContainer>
+                                )}
+                            </OptionLabel>
+                        ))}
+                    </OptionList>
+                </Dropdown>
+            )}
+        </Container>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Select/components.ts b/datahub-web-react/src/alchemy-components/components/Select/components.ts
new file mode 100644
index 0000000000000..a360238fef492
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/components.ts
@@ -0,0 +1,235 @@
+import { Button, Icon } from '@components';
+import { borders, colors, radius, shadows, spacing, transition, typography } from '@components/theme';
+import { Checkbox } from 'antd';
+import styled from 'styled-components';
+import { formLabelTextStyles, inputPlaceholderTextStyles, inputValueTextStyles } from '../commonStyles';
+import { SelectSizeOptions, SelectStyleProps } from './types';
+import { getOptionLabelStyle, getSelectFontStyles, getSelectStyle } from './utils';
+
+const sharedTransition = `${transition.property.colors} ${transition.easing['ease-in-out']} ${transition.duration.normal}`;
+
+/**
+ * Base Select component styling
+ */
+export const SelectBase = styled.div<SelectStyleProps>(({ isDisabled, isReadOnly, fontSize, isOpen }) => ({
+    ...getSelectStyle({ isDisabled, isReadOnly, fontSize, isOpen }),
+    display: 'flex',
+    flexDirection: 'row' as const,
+    gap: spacing.xsm,
+    transition: sharedTransition,
+    justifyContent: 'space-between',
+    alignItems: 'center',
+    overflow: 'auto',
+    backgroundColor: isDisabled ? colors.gray[100] : 'white',
+}));
+
+/**
+ * Styled components specific to the Basic version of the Select component
+ */
+
+// Container for the Basic Select component
+interface ContainerProps {
+    size: SelectSizeOptions;
+    width?: number | 'full';
+}
+
+export const Container = styled.div<ContainerProps>(({ size, width }) => ({
+    position: 'relative',
+    display: 'flex',
+    flexDirection: 'column',
+    width: width === 'full' ? '100%' : `${width}px`,
+    gap: '4px',
+    transition: sharedTransition,
+    minWidth: '175px',
+    ...getSelectFontStyles(size),
+    ...inputValueTextStyles(size),
+}));
+
+export const Dropdown = styled.div({
+    position: 'absolute',
+    top: '100%',
+    left: 0,
+    right: 0,
+    borderRadius: radius.md,
+    background: colors.white,
+    zIndex: 1,
+    transition: sharedTransition,
+    boxShadow: shadows.dropdown,
+    padding: spacing.xsm,
+    display: 'flex',
+    flexDirection: 'column',
+    gap: '8px',
+    marginTop: '4px',
+    maxHeight: '360px',
+    overflow: 'auto',
+});
+
+export const SearchInputContainer = styled.div({
+    position: 'relative',
+    width: '100%',
+    display: 'flex',
+    justifyContent: 'center',
+});
+
+export const SearchInput = styled.input({
+    width: '100%',
+    borderRadius: radius.md,
+    border: `1px solid ${colors.gray[200]}`,
+    color: colors.gray[500],
+    fontFamily: typography.fonts.body,
+    fontSize: typography.fontSizes.sm,
+    padding: spacing.xsm,
+    paddingRight: spacing.xlg,
+
+    '&:focus': {
+        borderColor: colors.violet[200],
+        outline: `${borders['1px']} ${colors.violet[200]}`,
+    },
+});
+
+export const SearchIcon = styled(Icon)({
+    position: 'absolute',
+    right: spacing.sm,
+    top: '50%',
+    transform: 'translateY(-50%)',
+    pointerEvents: 'none',
+});
+
+// Styled components for SelectValue (Selected value display)
+export const SelectValue = styled.span({
+    ...inputValueTextStyles(),
+});
+
+export const Placeholder = styled.span({
+    ...inputPlaceholderTextStyles,
+});
+
+export const ActionButtonsContainer = styled.div({
+    display: 'flex',
+    gap: '6px',
+    flexDirection: 'row',
+    alignItems: 'center',
+});
+
+/**
+ * Components that can be reused to create new Select variants
+ */
+
+export const FooterBase = styled.div({
+    display: 'flex',
+    justifyContent: 'flex-end',
+    gap: spacing.sm,
+    paddingTop: spacing.sm,
+    borderTop: `1px solid ${colors.gray[100]}`,
+});
+
+export const OptionList = styled.div({
+    display: 'flex',
+    flexDirection: 'column' as const,
+});
+
+export const LabelContainer = styled.div({
+    display: 'flex',
+    justifyContent: 'space-between',
+    width: '100%',
+});
+
+export const OptionContainer = styled.div({
+    display: 'flex',
+    flexDirection: 'column',
+});
+
+export const DescriptionContainer = styled.span({
+    whiteSpace: 'nowrap',
+    overflow: 'hidden',
+    textOverflow: 'ellipsis',
+    width: '100%',
+    color: colors.gray[500],
+    lineHeight: 'normal',
+    fontSize: typography.fontSizes.sm,
+    marginTop: spacing.xxsm,
+});
+
+export const LabelsWrapper = styled.div({
+    display: 'flex',
+    flexWrap: 'wrap',
+    gap: spacing.xxsm,
+    maxHeight: '150px',
+    maxWidth: 'calc(100% - 54px)',
+});
+
+export const OptionLabel = styled.label<{ isSelected: boolean; isMultiSelect?: boolean; isDisabled?: boolean }>(
+    ({ isSelected, isMultiSelect, isDisabled }) => ({
+        ...getOptionLabelStyle(isSelected, isMultiSelect, isDisabled),
+    }),
+);
+
+export const SelectAllOption = styled.div<{ isSelected: boolean; isDisabled?: boolean }>(
+    ({ isSelected, isDisabled }) => ({
+        cursor: isDisabled ? 'not-allowed' : 'pointer',
+        padding: spacing.xsm,
+        color: isSelected ? colors.violet[700] : colors.gray[500],
+        fontWeight: typography.fontWeights.semiBold,
+        fontSize: typography.fontSizes.md,
+        display: 'flex',
+        alignItems: 'center',
+    }),
+);
+
+export const SelectLabel = styled.label({
+    ...formLabelTextStyles,
+    marginBottom: spacing.xxsm,
+    textAlign: 'left',
+});
+
+export const StyledCancelButton = styled(Button)({
+    backgroundColor: colors.violet[100],
+    color: colors.violet[500],
+    borderColor: colors.violet[100],
+
+    '&:hover': {
+        backgroundColor: colors.violet[200],
+        borderColor: colors.violet[200],
+    },
+});
+
+export const StyledClearButton = styled(Button)({
+    backgroundColor: colors.gray[200],
+    border: `1px solid ${colors.gray[200]}`,
+    color: colors.black,
+    padding: '1px',
+
+    '&:hover': {
+        backgroundColor: colors.violet[100],
+        color: colors.violet[700],
+        borderColor: colors.violet[100],
+        boxShadow: shadows.none,
+    },
+
+    '&:focus': {
+        backgroundColor: colors.violet[100],
+        color: colors.violet[700],
+        boxShadow: `0 0 0 2px ${colors.white}, 0 0 0 4px ${colors.violet[50]}`,
+    },
+});
+
+export const ClearIcon = styled.span({
+    cursor: 'pointer',
+    marginLeft: '8px',
+});
+
+export const ArrowIcon = styled.span<{ isOpen: boolean }>(({ isOpen }) => ({
+    marginLeft: 'auto',
+    border: 'solid black',
+    borderWidth: '0 1px 1px 0',
+    display: 'inline-block',
+    padding: '3px',
+    transform: isOpen ? 'rotate(-135deg)' : 'rotate(45deg)',
+}));
+
+export const StyledCheckbox = styled(Checkbox)({
+    '.ant-checkbox-checked:not(.ant-checkbox-disabled) .ant-checkbox-inner': {
+        backgroundColor: colors.violet[500],
+        borderColor: `${colors.violet[500]} !important`,
+    },
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Select/index.ts b/datahub-web-react/src/alchemy-components/components/Select/index.ts
new file mode 100644
index 0000000000000..eb469d0edc004
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/index.ts
@@ -0,0 +1,3 @@
+export { Select, selectDefaults } from './Select';
+export { SimpleSelect } from './SimpleSelect';
+export type { SelectProps, SelectOption } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Select/types.ts b/datahub-web-react/src/alchemy-components/components/Select/types.ts
new file mode 100644
index 0000000000000..5ccde408b7699
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/types.ts
@@ -0,0 +1,61 @@
+export type SelectSizeOptions = 'sm' | 'md' | 'lg';
+
+export interface SelectOption {
+    value: string;
+    label: string;
+    description?: string;
+}
+
+export interface SelectProps {
+    options: SelectOption[];
+    label?: string;
+    values?: string[];
+    onCancel?: () => void;
+    onUpdate?: (selectedValues: string[]) => void;
+    size?: SelectSizeOptions;
+    showSearch?: boolean;
+    isDisabled?: boolean;
+    isReadOnly?: boolean;
+    isRequired?: boolean;
+    showClear?: boolean;
+    width?: number | 'full';
+    isMultiSelect?: boolean;
+    placeholder?: string;
+    disabledValues?: string[];
+    showSelectAll?: boolean;
+    selectAllLabel?: string;
+    optionListTestId?: string;
+    showDescriptions?: boolean;
+}
+
+export interface SelectStyleProps {
+    fontSize?: SelectSizeOptions;
+    isDisabled?: boolean;
+    isReadOnly?: boolean;
+    isRequired?: boolean;
+    isOpen?: boolean;
+}
+
+export interface ActionButtonsProps {
+    fontSize?: SelectSizeOptions;
+    selectedValues: string[];
+    isOpen: boolean;
+    isDisabled: boolean;
+    isReadOnly: boolean;
+    showClear: boolean;
+    handleClearSelection: () => void;
+}
+
+export interface SelectLabelDisplayProps {
+    selectedValues: string[];
+    options: SelectOption[];
+    placeholder: string;
+    isMultiSelect?: boolean;
+    removeOption?: (option: SelectOption) => void;
+    disabledValues?: string[];
+    showDescriptions?: boolean;
+}
+
+export interface SearchInputProps extends React.InputHTMLAttributes<HTMLInputElement> {
+    fontSize: SelectSizeOptions;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Select/utils.ts b/datahub-web-react/src/alchemy-components/components/Select/utils.ts
new file mode 100644
index 0000000000000..d054dd8ff737a
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Select/utils.ts
@@ -0,0 +1,125 @@
+import { borders, colors, radius, spacing, typography } from '@components/theme';
+import { getFontSize } from '@components/theme/utils';
+
+import { SelectStyleProps } from './types';
+
+export const getOptionLabelStyle = (isSelected: boolean, isMultiSelect?: boolean, isDisabled?: boolean) => ({
+    cursor: isDisabled ? 'not-allowed' : 'pointer',
+    padding: spacing.xsm,
+    borderRadius: radius.md,
+    lineHeight: typography.lineHeights.normal,
+    backgroundColor: isSelected && !isMultiSelect ? colors.violet[100] : 'transparent',
+    color: isSelected ? colors.violet[700] : colors.gray[500],
+    fontWeight: typography.fontWeights.medium,
+    fontSize: typography.fontSizes.md,
+    display: 'flex',
+    alignItems: 'center',
+
+    '&:hover': {
+        backgroundColor: isSelected ? colors.violet[100] : colors.gray[100],
+    },
+});
+
+export const getFooterButtonSize = (size) => {
+    return size === 'sm' ? 'sm' : 'md';
+};
+
+export const getSelectFontStyles = (size) => {
+    const baseFontStyles = {
+        lineHeight: typography.lineHeights.none,
+    };
+
+    const sizeStyles = {
+        sm: {
+            ...baseFontStyles,
+            fontSize: getFontSize(size),
+        },
+        md: {
+            ...baseFontStyles,
+            fontSize: getFontSize(size),
+        },
+        lg: {
+            ...baseFontStyles,
+            fontSize: getFontSize(size),
+        },
+    };
+
+    return sizeStyles[size];
+};
+
+export const getSelectPadding = (size) => {
+    const paddingStyles = {
+        sm: {
+            padding: `${spacing.sm} ${spacing.xsm}`,
+        },
+        md: {
+            padding: `${spacing.sm} ${spacing.md}`,
+        },
+        lg: {
+            padding: `${spacing.md} ${spacing.sm}`,
+        },
+    };
+
+    return paddingStyles[size];
+};
+
+export const getSearchPadding = (size) => {
+    const paddingStyles = {
+        sm: {
+            padding: `${spacing.xxsm} ${spacing.xsm}`,
+        },
+        md: {
+            padding: `${spacing.xsm} ${spacing.xsm}`,
+        },
+        lg: {
+            padding: `${spacing.xsm} ${spacing.xsm}`,
+        },
+    };
+
+    return paddingStyles[size];
+};
+
+export const getSelectStyle = (props: SelectStyleProps) => {
+    const { isDisabled, isReadOnly, fontSize, isOpen } = props;
+
+    const baseStyle = {
+        borderRadius: radius.md,
+        border: `1px solid ${colors.gray[200]}`,
+        fontFamily: typography.fonts.body,
+        color: isDisabled ? colors.gray[300] : colors.black,
+        cursor: isDisabled || isReadOnly ? 'not-allowed' : 'pointer',
+        backgroundColor: isDisabled ? colors.gray[100] : 'initial',
+
+        '&::placeholder': {
+            color: colors.gray[400],
+        },
+
+        // Open Styles
+        ...(isOpen
+            ? {
+                  borderColor: colors.violet[300],
+                  boxShadow: `0px 0px 4px 0px rgba(83, 63, 209, 0.5)`,
+                  outline: 'none',
+              }
+            : {}),
+
+        // Hover Styles
+        ...(isDisabled || isReadOnly || isOpen
+            ? {}
+            : {
+                  '&:hover': {
+                      borderColor: colors.violet[200],
+                      outline: `${borders['1px']} ${colors.violet[200]}`,
+                  },
+              }),
+    };
+
+    const fontStyles = getSelectFontStyles(fontSize);
+    const paddingStyles = getSelectPadding(fontSize);
+
+    return {
+        ...baseStyle,
+        ...fontStyles,
+        ...paddingStyles,
+    };
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Switch/Switch.stories.tsx b/datahub-web-react/src/alchemy-components/components/Switch/Switch.stories.tsx
new file mode 100644
index 0000000000000..7bb4ee2397cc6
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Switch/Switch.stories.tsx
@@ -0,0 +1,169 @@
+import React from 'react';
+
+import type { Meta, StoryObj } from '@storybook/react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+
+import { GridList } from '@components/.docs/mdx-components';
+
+import { Switch, switchDefaults } from './Switch';
+import { AVAILABLE_ICONS } from '../Icon';
+
+const meta = {
+    title: 'Forms / Switch',
+    component: Switch,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'A component that is used to get user input in the state of a toggle.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        label: {
+            description: 'Label for the Switch.',
+            table: {
+                defaultValue: { summary: switchDefaults.label },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        labelPosition: {
+            description: 'The position of the label relative to the Switch.',
+            options: ['left', 'top'],
+            table: {
+                defaultValue: { summary: switchDefaults.labelPosition },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        icon: {
+            description: 'The icon to display in the Switch Slider.',
+            type: 'string',
+            options: AVAILABLE_ICONS,
+            table: {
+                defaultValue: { summary: 'undefined' },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        colorScheme: {
+            description: 'The color of the Switch.',
+            options: ['violet', 'green', 'red', 'blue', 'gray'],
+            table: {
+                defaultValue: { summary: switchDefaults.colorScheme },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        size: {
+            description: 'The size of the Button.',
+            options: ['sm', 'md', 'lg', 'xl'],
+            table: {
+                defaultValue: { summary: switchDefaults.size },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        isSquare: {
+            description: 'Whether the Switch is square in shape.',
+            table: {
+                defaultValue: { summary: switchDefaults?.isSquare?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isChecked: {
+            description: 'Whether the Switch is checked.',
+            table: {
+                defaultValue: { summary: switchDefaults?.isChecked?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isDisabled: {
+            description: 'Whether the Switch is in disabled state.',
+            table: {
+                defaultValue: { summary: switchDefaults?.isDisabled?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isRequired: {
+            description: 'Whether the Switch is a required field.',
+            table: {
+                defaultValue: { summary: switchDefaults?.isRequired?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+    },
+
+    // Define defaults
+    args: {
+        label: switchDefaults.label,
+        labelPosition: switchDefaults.labelPosition,
+        icon: switchDefaults.icon,
+        colorScheme: switchDefaults.colorScheme,
+        size: switchDefaults.size,
+        isSquare: switchDefaults.isSquare,
+        isChecked: switchDefaults.isChecked,
+        isDisabled: switchDefaults.isDisabled,
+        isRequired: switchDefaults.isRequired,
+    },
+} satisfies Meta<typeof Switch>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Switch {...props} />,
+};
+
+export const sizes = () => (
+    <GridList isVertical>
+        <Switch label="Small" size="sm" />
+        <Switch label="Medium" size="md" />
+        <Switch label="Large" size="lg" />
+        <Switch label="XLarge" size="xl" />
+    </GridList>
+);
+
+export const colors = () => (
+    <GridList isVertical>
+        <Switch label="Default" isChecked colorScheme="violet" />
+        <Switch label="Green" isChecked colorScheme="green" />
+        <Switch label="Red" isChecked colorScheme="red" />
+        <Switch label="Blue" isChecked colorScheme="blue" />
+        <Switch label="Gray" isChecked colorScheme="gray" />
+    </GridList>
+);
+
+export const states = () => (
+    <GridList isVertical>
+        <Switch label="Disabled" isDisabled />
+        <Switch label="Required" isRequired />
+        <Switch label="with Icon" icon="Add" />
+    </GridList>
+);
+
+export const types = () => (
+    <GridList width="400px">
+        <Switch label="Default" />
+        <Switch label="Square" isSquare />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Switch/Switch.tsx b/datahub-web-react/src/alchemy-components/components/Switch/Switch.tsx
new file mode 100644
index 0000000000000..18a01386562ee
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Switch/Switch.tsx
@@ -0,0 +1,74 @@
+import { Tooltip } from '@components';
+import React, { useEffect, useState } from 'react';
+import { IconContainer, Label, Required, Slider, StyledIcon, StyledInput, SwitchContainer } from './components';
+import { SwitchProps } from './types';
+
+export const switchDefaults: SwitchProps = {
+    label: 'Label',
+    labelPosition: 'left',
+    colorScheme: 'violet',
+    size: 'md',
+    isSquare: false,
+    isChecked: false,
+    isDisabled: false,
+    isRequired: false,
+};
+
+export const Switch = ({
+    label = switchDefaults.label,
+    labelPosition = switchDefaults.labelPosition,
+    icon, // undefined by default
+    colorScheme = switchDefaults.colorScheme,
+    size = switchDefaults.size,
+    isSquare = switchDefaults.isSquare,
+    isChecked = switchDefaults.isChecked,
+    isDisabled = switchDefaults.isDisabled,
+    isRequired = switchDefaults.isRequired,
+    labelHoverText,
+    disabledHoverText,
+    labelStyle,
+    ...props
+}: SwitchProps) => {
+    const [checked, setChecked] = useState(isChecked);
+
+    useEffect(() => {
+        setChecked(isChecked);
+    }, [isChecked]);
+
+    const id = props.id || `switchToggle-${label}`;
+
+    return (
+        <SwitchContainer labelPosition={labelPosition || 'left'} isDisabled={isDisabled}>
+            <Tooltip title={labelHoverText} showArrow={false}>
+                <Label id={id} aria-label={label} style={labelStyle}>
+                    {label} {isRequired && <Required>*</Required>}
+                </Label>
+            </Tooltip>
+            <StyledInput
+                type="checkbox"
+                checked={checked}
+                onChange={() => setChecked(!checked)}
+                customSize={size}
+                disabled={isDisabled}
+                colorScheme={colorScheme || 'violet'}
+                aria-labelledby={id}
+                aria-checked={checked}
+                {...props}
+            />
+            <Tooltip title={isDisabled && disabledHoverText ? disabledHoverText : undefined} showArrow={false}>
+                <Slider size={size} isSquare={isSquare} isDisabled={isDisabled}>
+                    <IconContainer>
+                        {icon && (
+                            <StyledIcon
+                                icon={icon}
+                                color={checked ? colorScheme || 'violet' : 'inherit'}
+                                size={size || 'md'}
+                                checked={checked}
+                            />
+                        )}
+                    </IconContainer>
+                </Slider>
+            </Tooltip>
+        </SwitchContainer>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Switch/components.ts b/datahub-web-react/src/alchemy-components/components/Switch/components.ts
new file mode 100644
index 0000000000000..1586c1cf9f32f
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Switch/components.ts
@@ -0,0 +1,118 @@
+import styled from 'styled-components';
+
+import { borders, colors, shadows, spacing, transition } from '@components/theme';
+import { ColorOptions, SizeOptions } from '@components/theme/config';
+
+import { Icon } from '../Icon';
+
+import { formLabelTextStyles } from '../commonStyles';
+
+import {
+    getIconTransformPositionLeft,
+    getIconTransformPositionTop,
+    getInputHeight,
+    getSliderTransformPosition,
+    getToggleSize,
+} from './utils';
+
+import type { SwitchLabelPosition } from './types';
+
+export const Label = styled.div({
+    ...formLabelTextStyles,
+    display: 'flex',
+    alignItems: 'flex-start',
+});
+
+export const SwitchContainer = styled.label<{ labelPosition: SwitchLabelPosition; isDisabled?: boolean }>(
+    ({ labelPosition, isDisabled }) => ({
+        display: 'flex',
+        flexDirection: labelPosition === 'top' ? 'column' : 'row',
+        alignItems: labelPosition === 'top' ? 'flex-start' : 'center',
+        gap: spacing.sm,
+        cursor: isDisabled ? 'not-allowed' : 'pointer',
+        width: 'max-content',
+    }),
+);
+
+export const Slider = styled.div<{ size?: SizeOptions; isSquare?: boolean; isDisabled?: boolean }>(
+    ({ size, isSquare, isDisabled }) => ({
+        '&:before': {
+            transition: `${transition.duration.normal} all`,
+            content: '""',
+            position: 'absolute',
+            minWidth: getToggleSize(size || 'md', 'slider'), // sliders width and height must be same
+            minHeight: getToggleSize(size || 'md', 'slider'),
+            borderRadius: !isSquare ? '35px' : '0px',
+            top: '50%',
+            left: spacing.xxsm,
+            transform: 'translate(0, -50%)',
+            backgroundColor: !isDisabled ? colors.white : colors.gray[200],
+            boxShadow: `
+				0px 1px 2px 0px rgba(16, 24, 40, 0.06),
+				0px 1px 3px 0px rgba(16, 24, 40, 0.12)
+			`,
+        },
+        borderRadius: !isSquare ? '32px' : '0px',
+        minWidth: getToggleSize(size || 'md', 'input'),
+        minHeight: getInputHeight(size || 'md'),
+    }),
+    {
+        display: 'flex',
+        justifyContent: 'center',
+        alignItems: 'center',
+        position: 'relative',
+
+        backgroundColor: colors.gray[100],
+        padding: spacing.xxsm,
+        transition: `${transition.duration.normal} all`,
+        boxSizing: 'content-box',
+    },
+);
+
+export const Required = styled.span({
+    color: colors.red[500],
+    marginLeft: spacing.xxsm,
+});
+
+export const StyledInput = styled.input<{
+    customSize?: SizeOptions;
+    disabled?: boolean;
+    colorScheme: ColorOptions;
+    checked?: boolean;
+}>`
+    opacity: 0;
+    position: absolute;
+
+    &:checked + ${Slider} {
+        background-color: ${(props) => (!props.disabled ? colors[props.colorScheme][500] : colors.gray[100])};
+
+        &:before {
+            transform: ${({ customSize }) => getSliderTransformPosition(customSize || 'md')};
+        }
+    }
+
+    &:focus-within + ${Slider} {
+        border-color: ${(props) => (props.checked ? colors[props.colorScheme][200] : 'transparent')};
+        outline: ${(props) => (props.checked ? `${borders['2px']} ${colors[props.colorScheme][200]}` : 'none')};
+        box-shadow: ${(props) => (props.checked ? shadows.xs : 'none')};
+    }
+`;
+
+export const StyledIcon = styled(Icon)<{ checked?: boolean; size: SizeOptions }>(
+    ({ checked, size }) => ({
+        left: getIconTransformPositionLeft(size, checked || false),
+        top: getIconTransformPositionTop(size),
+    }),
+    {
+        transition: `${transition.duration.normal} all`,
+        position: 'absolute',
+        display: 'flex',
+        alignItems: 'center',
+        justifyContent: 'center',
+        color: colors.gray[500],
+    },
+);
+
+export const IconContainer = styled.div({
+    position: 'relative',
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Switch/index.ts b/datahub-web-react/src/alchemy-components/components/Switch/index.ts
new file mode 100644
index 0000000000000..0c48d2964887e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Switch/index.ts
@@ -0,0 +1,2 @@
+export { Switch, switchDefaults } from './Switch';
+export type { SwitchProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Switch/types.ts b/datahub-web-react/src/alchemy-components/components/Switch/types.ts
new file mode 100644
index 0000000000000..e15c0f81b4a39
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Switch/types.ts
@@ -0,0 +1,21 @@
+import { ColorOptions, SizeOptions } from '@components/theme/config';
+import { InputHTMLAttributes } from 'react';
+import { CSSProperties } from 'styled-components';
+import { IconNames } from '../Icon';
+
+export type SwitchLabelPosition = 'left' | 'top';
+
+export interface SwitchProps extends Omit<InputHTMLAttributes<HTMLInputElement>, 'size'> {
+    label: string;
+    labelPosition?: SwitchLabelPosition;
+    icon?: IconNames;
+    colorScheme?: ColorOptions;
+    size?: SizeOptions;
+    isSquare?: boolean;
+    isChecked?: boolean;
+    isDisabled?: boolean;
+    isRequired?: boolean;
+    labelHoverText?: string;
+    disabledHoverText?: string;
+    labelStyle?: CSSProperties;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Switch/utils.ts b/datahub-web-react/src/alchemy-components/components/Switch/utils.ts
new file mode 100644
index 0000000000000..c0365baa34818
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Switch/utils.ts
@@ -0,0 +1,97 @@
+import { SizeOptions } from '@components/theme/config';
+
+const sliderSize = {
+    sm: '14px',
+    md: '16px',
+    lg: '18px',
+    xl: '20px',
+};
+
+const inputSize = {
+    sm: '35px',
+    md: '40px',
+    lg: '45px',
+    xl: '50px',
+};
+
+const translateSize = {
+    sm: '22px',
+    md: '24px',
+    lg: '26px',
+    xl: '28px',
+};
+
+const iconTransformPositionLeft = {
+    sm: {
+        checked: '5.5px',
+        unchecked: '-16.5px',
+    },
+    md: {
+        checked: '5px',
+        unchecked: '-19px',
+    },
+    lg: {
+        checked: '4.5px',
+        unchecked: '-21.5px',
+    },
+    xl: {
+        checked: '4px',
+        unchecked: '-24px',
+    },
+};
+
+const iconTransformPositionTop = {
+    sm: '-6px',
+    md: '-7px',
+    lg: '-8px',
+    xl: '-9px',
+};
+
+export const getToggleSize = (size: SizeOptions, mode: 'slider' | 'input'): string => {
+    if (size === 'sm') return mode === 'slider' ? sliderSize.sm : inputSize.sm;
+    if (size === 'md') return mode === 'slider' ? sliderSize.md : inputSize.md;
+    if (size === 'lg') return mode === 'slider' ? sliderSize.lg : inputSize.lg;
+    return mode === 'slider' ? sliderSize.xl : inputSize.xl; // xl
+};
+
+export const getInputHeight = (size: SizeOptions) => {
+    if (size === 'sm') return sliderSize.sm;
+    if (size === 'md') return sliderSize.md;
+    if (size === 'lg') return sliderSize.lg;
+    return sliderSize.xl; // xl
+};
+
+export const getSliderTransformPosition = (size: SizeOptions): string => {
+    if (size === 'sm') return `translate(${translateSize.sm}, -50%)`;
+    if (size === 'md') return `translate(${translateSize.md}, -50%)`;
+    if (size === 'lg') return `translate(${translateSize.lg}, -50%)`;
+    return `translate(${translateSize.xl}, -50%)`; // xl
+};
+
+export const getIconTransformPositionLeft = (size: SizeOptions, checked: boolean): string => {
+    if (size === 'sm') {
+        if (checked) return iconTransformPositionLeft.sm.checked;
+        return iconTransformPositionLeft.sm.unchecked;
+    }
+
+    if (size === 'md') {
+        if (checked) return iconTransformPositionLeft.md.checked;
+        return iconTransformPositionLeft.md.unchecked;
+    }
+
+    if (size === 'lg') {
+        if (checked) return iconTransformPositionLeft.lg.checked;
+        return iconTransformPositionLeft.lg.unchecked;
+    }
+
+    // xl
+    if (checked) return iconTransformPositionLeft.xl.checked;
+    return iconTransformPositionLeft.xl.unchecked;
+};
+
+export const getIconTransformPositionTop = (size: SizeOptions): string => {
+    if (size === 'sm') return iconTransformPositionTop.sm;
+    if (size === 'md') return iconTransformPositionTop.md;
+    if (size === 'lg') return iconTransformPositionTop.lg;
+    return iconTransformPositionTop.xl; // xl
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Table/Table.stories.tsx b/datahub-web-react/src/alchemy-components/components/Table/Table.stories.tsx
new file mode 100644
index 0000000000000..3a36b65897806
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Table/Table.stories.tsx
@@ -0,0 +1,162 @@
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import { Meta, StoryObj } from '@storybook/react';
+import React from 'react';
+import { Table, tableDefaults } from '.';
+
+// Auto Docs
+const meta = {
+    title: 'Lists & Tables / Table',
+    component: Table,
+
+    // Display Properties
+    parameters: {
+        layout: 'padded',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'This component allows users to render a table with different columns and their data',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        columns: {
+            description: 'Array of column objects for the table header.',
+            control: 'object',
+            table: {
+                defaultValue: { summary: JSON.stringify(tableDefaults.columns) },
+            },
+        },
+        data: {
+            description: 'Array of data rows for the table body.',
+            control: 'object',
+            table: {
+                defaultValue: { summary: JSON.stringify(tableDefaults.data) },
+            },
+        },
+        showHeader: {
+            description: 'Whether to show the table header.',
+            control: 'boolean',
+            table: {
+                defaultValue: { summary: tableDefaults.showHeader?.toString() },
+            },
+        },
+        isLoading: {
+            description: 'Whether the table is in loading state.',
+            control: 'boolean',
+            table: {
+                defaultValue: { summary: tableDefaults.isLoading?.toString() },
+            },
+        },
+        isScrollable: {
+            description: 'Whether the table is scrollable.',
+            control: 'boolean',
+            table: {
+                defaultValue: { summary: tableDefaults.isScrollable?.toString() },
+            },
+        },
+        maxHeight: {
+            description: 'Maximum height of the table container.',
+            control: 'text',
+            table: {
+                defaultValue: { summary: tableDefaults.maxHeight },
+            },
+        },
+    },
+
+    // Define defaults
+    args: {
+        columns: [
+            { title: 'Column 1', key: 'column1', dataIndex: 'column1' },
+            { title: 'Column 2', key: 'column2', dataIndex: 'column2' },
+        ],
+        data: [
+            { column1: 'Row 1 Col 1', column2: 'Row 1 Col 2' },
+            { column1: 'Row 2 Col 1', column2: 'Row 2 Col 2' },
+        ],
+        showHeader: tableDefaults.showHeader,
+        isLoading: tableDefaults.isLoading,
+        isScrollable: tableDefaults.isScrollable,
+        maxHeight: tableDefaults.maxHeight,
+    },
+} satisfies Meta<typeof Table<any>>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+// Basic story is what is displayed 1st in storybook & is used as the code sandbox
+// Pass props to this so that it can be customized via the UI props panel
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Table {...props} />,
+};
+
+export const withScroll = () => (
+    <Table
+        columns={[
+            { title: 'Column 1', key: 'column1', dataIndex: 'column1' },
+            { title: 'Column 2', key: 'column2', dataIndex: 'column2' },
+        ]}
+        data={[
+            { column1: 'Row 1 Col 1', column2: 'Row 1 Col 2' },
+            { column1: 'Row 2 Col 1', column2: 'Row 2 Col 2' },
+            { column1: 'Row 3 Col 1', column2: 'Row 3 Col 2' },
+            { column1: 'Row 4 Col 1', column2: 'Row 4 Col 2' },
+            { column1: 'Row 5 Col 1', column2: 'Row 5 Col 2' },
+            { column1: 'Row 6 Col 1', column2: 'Row 6 Col 2' },
+        ]}
+        maxHeight="180px"
+        isScrollable
+    />
+);
+
+export const withCustomColumnWidths = () => (
+    <Table
+        columns={[
+            { title: 'Column 1', key: 'column1', dataIndex: 'column1', width: '50%' },
+            { title: 'Column 2', key: 'column2', dataIndex: 'column2', width: '30%' },
+            { title: 'Column 3', key: 'column3', dataIndex: 'column3', width: '20%' },
+        ]}
+        data={[
+            { column1: 'Row 1 Col 1', column2: 'Row 1 Col 2', column3: 1 },
+            { column1: 'Row 2 Col 1', column2: 'Row 2 Col 2', column3: 2 },
+            { column1: 'Row 3 Col 1', column2: 'Row 3 Col 2', column3: 3 },
+        ]}
+    />
+);
+
+export const withColumnSorting = () => (
+    <Table
+        columns={[
+            {
+                title: 'Column 1',
+                key: 'column1',
+                dataIndex: 'column1',
+                sorter: (a, b) => a.column1.localeCompare(b.column1),
+            },
+            { title: 'Column 2', key: 'column2', dataIndex: 'column2' },
+            { title: 'Column 3', key: 'column3', dataIndex: 'column3', sorter: (a, b) => a.column3 - b.column3 },
+        ]}
+        data={[
+            { column1: 'Row 2 Col 1', column2: 'Row 2 Col 2', column3: 3 },
+            { column1: 'Row 1 Col 1', column2: 'Row 1 Col 2', column3: 2 },
+            { column1: 'Row 3 Col 1', column2: 'Row 3 Col 2', column3: 1 },
+        ]}
+    />
+);
+
+export const withoutHeader = () => (
+    <Table
+        columns={[
+            { title: 'Column 1', key: 'column1', dataIndex: 'column1' },
+            { title: 'Column 2', key: 'column2', dataIndex: 'column2' },
+        ]}
+        data={[
+            { column1: 'Row 1 Col 1', column2: 'Row 1 Col 2' },
+            { column1: 'Row 2 Col 1', column2: 'Row 2 Col 2' },
+        ]}
+        showHeader={false}
+    />
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Table/Table.tsx b/datahub-web-react/src/alchemy-components/components/Table/Table.tsx
new file mode 100644
index 0000000000000..11e598f8d4e0f
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Table/Table.tsx
@@ -0,0 +1,115 @@
+import { LoadingOutlined } from '@ant-design/icons';
+import { Text } from '@components';
+import React, { useState } from 'react';
+import {
+    BaseTable,
+    HeaderContainer,
+    LoadingContainer,
+    SortIcon,
+    SortIconsContainer,
+    TableCell,
+    TableContainer,
+    TableHeader,
+    TableHeaderCell,
+    TableRow,
+} from './components';
+import { TableProps } from './types';
+import { getSortedData, handleActiveSort, renderCell, SortingState } from './utils';
+
+export const tableDefaults: TableProps<any> = {
+    columns: [],
+    data: [],
+    showHeader: true,
+    isLoading: false,
+    isScrollable: false,
+    maxHeight: '100%',
+};
+
+export const Table = <T,>({
+    columns = tableDefaults.columns,
+    data = tableDefaults.data,
+    showHeader = tableDefaults.showHeader,
+    isLoading = tableDefaults.isLoading,
+    isScrollable = tableDefaults.isScrollable,
+    maxHeight = tableDefaults.maxHeight,
+    ...props
+}: TableProps<T>) => {
+    const [sortColumn, setSortColumn] = useState<string | null>(null);
+    const [sortOrder, setSortOrder] = useState<SortingState>(SortingState.ORIGINAL);
+
+    const sortedData = getSortedData(columns, data, sortColumn, sortOrder);
+
+    if (isLoading) {
+        return (
+            <LoadingContainer>
+                <LoadingOutlined />
+                <Text color="gray">Loading data...</Text>
+            </LoadingContainer>
+        );
+    }
+
+    return (
+        <TableContainer isScrollable={isScrollable} maxHeight={maxHeight}>
+            <BaseTable {...props}>
+                {showHeader && (
+                    <TableHeader>
+                        <tr>
+                            {columns.map((column) => (
+                                <TableHeaderCell key={column.key} width={column.width}>
+                                    <HeaderContainer>
+                                        {column.title}
+                                        {column.sorter && (
+                                            <SortIconsContainer
+                                                onClick={() =>
+                                                    column.sorter &&
+                                                    handleActiveSort(
+                                                        column.key,
+                                                        sortColumn,
+                                                        setSortColumn,
+                                                        setSortOrder,
+                                                    )
+                                                }
+                                            >
+                                                <SortIcon
+                                                    icon="ChevronLeft"
+                                                    size="md"
+                                                    rotate="90"
+                                                    isActive={
+                                                        column.key === sortColumn &&
+                                                        sortOrder === SortingState.ASCENDING
+                                                    }
+                                                />
+                                                <SortIcon
+                                                    icon="ChevronRight"
+                                                    size="md"
+                                                    rotate="90"
+                                                    isActive={
+                                                        column.key === sortColumn &&
+                                                        sortOrder === SortingState.DESCENDING
+                                                    }
+                                                />
+                                            </SortIconsContainer>
+                                        )}
+                                    </HeaderContainer>
+                                </TableHeaderCell>
+                            ))}
+                        </tr>
+                    </TableHeader>
+                )}
+                <tbody>
+                    {sortedData.map((row, index) => (
+                        <TableRow>
+                            {columns.map((column) => {
+                                return (
+                                    <TableCell key={column.key} width={column.width} alignment={column.alignment}>
+                                        {renderCell(column, row, index)}
+                                    </TableCell>
+                                );
+                            })}
+                        </TableRow>
+                    ))}
+                </tbody>
+            </BaseTable>
+        </TableContainer>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Table/components.ts b/datahub-web-react/src/alchemy-components/components/Table/components.ts
new file mode 100644
index 0000000000000..8908256a81ddf
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Table/components.ts
@@ -0,0 +1,94 @@
+import { Icon } from '@components';
+import { colors, radius, spacing, typography } from '@src/alchemy-components/theme';
+import { AlignmentOptions } from '@src/alchemy-components/theme/config';
+import styled from 'styled-components';
+
+export const TableContainer = styled.div<{ isScrollable?: boolean; maxHeight?: string }>(
+    ({ isScrollable, maxHeight }) => ({
+        borderRadius: radius.lg,
+        border: `1px solid ${colors.gray[1400]}`,
+        overflow: isScrollable ? 'auto' : 'hidden',
+        width: '100%',
+        maxHeight: maxHeight || '100%',
+    }),
+);
+
+export const BaseTable = styled.table({
+    borderCollapse: 'collapse',
+    width: '100%',
+});
+
+export const TableHeader = styled.thead({
+    backgroundColor: colors.gray[1500],
+    borderRadius: radius.lg,
+    position: 'sticky',
+    top: 0,
+    zIndex: 100,
+});
+
+export const TableHeaderCell = styled.th<{ width?: string }>(({ width }) => ({
+    padding: `${spacing.sm} ${spacing.md}`,
+    color: colors.gray[600],
+    fontSize: typography.fontSizes.sm,
+    fontWeight: typography.fontWeights.medium,
+    textAlign: 'start',
+    width: width || 'auto',
+}));
+
+export const HeaderContainer = styled.div({
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing.sm,
+});
+
+export const TableRow = styled.tr({
+    '&:last-child': {
+        '& td': {
+            borderBottom: 'none',
+        },
+    },
+
+    '& td:first-child': {
+        fontWeight: typography.fontWeights.medium,
+        color: colors.gray[600],
+    },
+});
+
+export const TableCell = styled.td<{ width?: string; alignment?: AlignmentOptions }>(({ width, alignment }) => ({
+    padding: spacing.md,
+    borderBottom: `1px solid ${colors.gray[1400]}`,
+    color: colors.gray[1700],
+    fontSize: typography.fontSizes.md,
+    fontWeight: typography.fontWeights.normal,
+    overflow: 'hidden',
+    textOverflow: 'ellipsis',
+    whiteSpace: 'nowrap',
+    maxWidth: width || 'unset',
+    textAlign: alignment || 'left',
+}));
+
+export const SortIconsContainer = styled.div({
+    display: 'flex',
+    flexDirection: 'column',
+});
+
+export const SortIcon = styled(Icon)<{ isActive?: boolean }>(({ isActive }) => ({
+    margin: '-3px',
+    stroke: isActive ? colors.violet[600] : undefined,
+
+    ':hover': {
+        cursor: 'pointer',
+    },
+}));
+
+export const LoadingContainer = styled.div({
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'center',
+    justifyContent: 'center',
+    height: '100%',
+    width: '100%',
+    gap: spacing.sm,
+    color: colors.violet[700],
+    fontSize: typography.fontSizes['3xl'],
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Table/index.ts b/datahub-web-react/src/alchemy-components/components/Table/index.ts
new file mode 100644
index 0000000000000..986f467da74b8
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Table/index.ts
@@ -0,0 +1,2 @@
+export { Table, tableDefaults } from './Table';
+export type { Column, TableProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Table/types.ts b/datahub-web-react/src/alchemy-components/components/Table/types.ts
new file mode 100644
index 0000000000000..b3e0357d5cf14
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Table/types.ts
@@ -0,0 +1,21 @@
+import { AlignmentOptions } from '@src/alchemy-components/theme/config';
+import { TableHTMLAttributes } from 'react';
+
+export interface Column<T> {
+    title: string;
+    key: string;
+    dataIndex?: string;
+    render?: (record: T, index: number) => React.ReactNode;
+    width?: string;
+    sorter?: (a: T, b: T) => number;
+    alignment?: AlignmentOptions;
+}
+
+export interface TableProps<T> extends TableHTMLAttributes<HTMLTableElement> {
+    columns: Column<T>[];
+    data: T[];
+    showHeader?: boolean;
+    isLoading?: boolean;
+    isScrollable?: boolean;
+    maxHeight?: string;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Table/utils.ts b/datahub-web-react/src/alchemy-components/components/Table/utils.ts
new file mode 100644
index 0000000000000..c76494d32ca63
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Table/utils.ts
@@ -0,0 +1,73 @@
+import { Column } from './types';
+
+export enum SortingState {
+    ASCENDING = 'ascending',
+    DESCENDING = 'descending',
+    ORIGINAL = 'original',
+}
+
+export const handleActiveSort = (
+    key: string,
+    sortColumn: string | null,
+    setSortColumn: React.Dispatch<React.SetStateAction<string | null>>,
+    setSortOrder: React.Dispatch<React.SetStateAction<SortingState>>,
+) => {
+    if (sortColumn === key) {
+        // Toggle sort order
+        setSortOrder((prevOrder) => {
+            if (prevOrder === SortingState.ASCENDING) return SortingState.DESCENDING;
+            if (prevOrder === SortingState.DESCENDING) return SortingState.ORIGINAL;
+            return SortingState.ASCENDING;
+        });
+    } else {
+        // Set new column and default sort order
+        setSortColumn(key);
+        setSortOrder(SortingState.ASCENDING);
+    }
+};
+
+export const getSortedData = <T>(
+    columns: Column<T>[],
+    data: T[],
+    sortColumn: string | null,
+    sortOrder: SortingState,
+) => {
+    if (sortOrder === SortingState.ORIGINAL || !sortColumn) {
+        return data;
+    }
+
+    const activeColumn = columns.find((column) => column.key === sortColumn);
+
+    // Sort based on the order and column sorter
+    if (activeColumn && activeColumn.sorter) {
+        return data.slice().sort((a, b) => {
+            return sortOrder === SortingState.ASCENDING ? activeColumn.sorter!(a, b) : activeColumn.sorter!(b, a);
+        });
+    }
+
+    return data;
+};
+
+export const renderCell = <T>(column: Column<T>, row: T, index: number) => {
+    const { render, dataIndex } = column;
+
+    let cellData;
+
+    if (dataIndex) {
+        cellData = row[dataIndex];
+
+        if (typeof dataIndex === 'string') {
+            cellData = dataIndex.split('.').reduce((acc, prop) => acc && acc[prop], row);
+        }
+
+        if (Array.isArray(dataIndex)) {
+            cellData = dataIndex.reduce((acc, prop) => acc && acc[prop], row);
+        }
+    }
+
+    if (render) {
+        return render(row, index);
+    }
+
+    return cellData;
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Text/Text.stories.tsx b/datahub-web-react/src/alchemy-components/components/Text/Text.stories.tsx
new file mode 100644
index 0000000000000..c82d468aaa08c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Text/Text.stories.tsx
@@ -0,0 +1,100 @@
+import React from 'react';
+
+import type { Meta, StoryObj, StoryFn } from '@storybook/react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+
+import { VerticalFlexGrid } from '@components/.docs/mdx-components';
+import { Text, textDefaults } from '.';
+
+// Auto Docs
+const meta = {
+    title: 'Typography / Text',
+    component: Text,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'Used to render text and paragraphs within an interface.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        children: {
+            description: 'The content to display within the heading.',
+            table: {
+                type: { summary: 'string' },
+            },
+        },
+        type: {
+            description: 'The type of text to display.',
+            table: {
+                defaultValue: { summary: textDefaults.type },
+            },
+        },
+        size: {
+            description: 'Override the size of the text.',
+            table: {
+                defaultValue: { summary: `${textDefaults.size}` },
+            },
+        },
+        color: {
+            description: 'Override the color of the text.',
+            table: {
+                defaultValue: { summary: textDefaults.color },
+            },
+        },
+        weight: {
+            description: 'Override the weight of the heading.',
+            table: {
+                defaultValue: { summary: textDefaults.weight },
+            },
+        },
+    },
+
+    // Define default args
+    args: {
+        children:
+            'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas aliquet nulla id felis vehicula, et posuere dui dapibus. Nullam rhoncus massa non tortor convallis, in blandit turpis rutrum. Morbi tempus velit mauris, at mattis metus mattis sed. Nunc molestie efficitur lectus, vel mollis eros.',
+        type: textDefaults.type,
+        size: textDefaults.size,
+        color: textDefaults.color,
+        weight: textDefaults.weight,
+    },
+} satisfies Meta<typeof Text>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+// Basic story is what is displayed 1st in storybook
+// Pass props to this so that it can be customized via the UI props panel
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <Text {...props}>{props.children}</Text>,
+};
+
+export const sizes: StoryFn<Story> = (props: any) => (
+    <VerticalFlexGrid>
+        <Text size="4xl">{props.children}</Text>
+        <Text size="3xl">{props.children}</Text>
+        <Text size="2xl">{props.children}</Text>
+        <Text size="xl">{props.children}</Text>
+        <Text size="lg">{props.children}</Text>
+        <Text size="md">{props.children}</Text>
+        <Text size="sm">{props.children}</Text>
+        <Text size="xs">{props.children}</Text>
+    </VerticalFlexGrid>
+);
+
+export const withLink = () => (
+    <Text>
+        Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas aliquet nulla id felis vehicula, et posuere
+        dui dapibus. <a href="/">Nullam rhoncus massa non tortor convallis</a>, in blandit turpis rutrum. Morbi tempus
+        velit mauris, at mattis metus mattis sed. Nunc molestie efficitur lectus, vel mollis eros.
+    </Text>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Text/Text.tsx b/datahub-web-react/src/alchemy-components/components/Text/Text.tsx
new file mode 100644
index 0000000000000..89122afbfcc8b
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Text/Text.tsx
@@ -0,0 +1,33 @@
+import React from 'react';
+
+import { TextProps } from './types';
+import { P, Div, Span } from './components';
+
+export const textDefaults: TextProps = {
+    type: 'p',
+    color: 'inherit',
+    size: 'md',
+    weight: 'normal',
+};
+
+export const Text = ({
+    type = textDefaults.type,
+    color = textDefaults.color,
+    size = textDefaults.size,
+    weight = textDefaults.weight,
+    children,
+    ...props
+}: TextProps) => {
+    const sharedProps = { size, color, weight, ...props };
+
+    switch (type) {
+        case 'p':
+            return <P {...sharedProps}>{children}</P>;
+        case 'div':
+            return <Div {...sharedProps}>{children}</Div>;
+        case 'span':
+            return <Span {...sharedProps}>{children}</Span>;
+        default:
+            return <P {...sharedProps}>{children}</P>;
+    }
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Text/components.ts b/datahub-web-react/src/alchemy-components/components/Text/components.ts
new file mode 100644
index 0000000000000..1d48497f39c9c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Text/components.ts
@@ -0,0 +1,50 @@
+import styled from 'styled-components';
+
+import { typography, colors } from '@components/theme';
+import { getColor, getFontSize } from '@components/theme/utils';
+import { TextProps } from './types';
+
+// Text Styles
+const textStyles = {
+    fontSize: typography.fontSizes.md,
+    lineHeight: typography.lineHeights.md,
+    fontWeight: typography.fontWeights.normal,
+};
+
+// Default styles
+const baseStyles = {
+    fontFamily: typography.fonts.body,
+    margin: 0,
+
+    '& a': {
+        color: colors.violet[400],
+        textDecoration: 'none',
+        transition: 'color 0.15s ease',
+
+        '&:hover': {
+            color: colors.violet[500],
+        },
+    },
+};
+
+// Prop Driven Styles
+const propStyles = (props, isText = false) => {
+    const styles = {} as any;
+    if (props.size) styles.fontSize = getFontSize(props.size);
+    if (props.color) styles.color = getColor(props.color);
+    if (props.weight) styles.fontWeight = typography.fontWeights[props.weight];
+    if (isText) styles.lineHeight = typography.lineHeights[props.size || 'md'];
+    return styles;
+};
+
+export const P = styled.p({ ...baseStyles, ...textStyles }, (props: TextProps) => ({
+    ...propStyles(props as TextProps, true),
+}));
+
+export const Span = styled.span({ ...baseStyles, ...textStyles }, (props: TextProps) => ({
+    ...propStyles(props as TextProps, true),
+}));
+
+export const Div = styled.div({ ...baseStyles, ...textStyles }, (props: TextProps) => ({
+    ...propStyles(props as TextProps, true),
+}));
diff --git a/datahub-web-react/src/alchemy-components/components/Text/index.ts b/datahub-web-react/src/alchemy-components/components/Text/index.ts
new file mode 100644
index 0000000000000..d4240105173d4
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Text/index.ts
@@ -0,0 +1,2 @@
+export { Text, textDefaults } from './Text';
+export type { TextProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/Text/types.ts b/datahub-web-react/src/alchemy-components/components/Text/types.ts
new file mode 100644
index 0000000000000..6a41929da12a9
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Text/types.ts
@@ -0,0 +1,9 @@
+import { HTMLAttributes } from 'react';
+import type { FontSizeOptions, FontColorOptions, FontWeightOptions } from '@components/theme/config';
+
+export interface TextProps extends HTMLAttributes<HTMLElement> {
+    type?: 'span' | 'p' | 'div';
+    size?: FontSizeOptions;
+    color?: FontColorOptions;
+    weight?: FontWeightOptions;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.stories.tsx b/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.stories.tsx
new file mode 100644
index 0000000000000..b244eefa6f207
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.stories.tsx
@@ -0,0 +1,159 @@
+import React from 'react';
+
+import type { Meta, StoryObj } from '@storybook/react';
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+
+import { GridList } from '@components/.docs/mdx-components';
+
+import { TextArea, textAreaDefaults } from './TextArea';
+import { AVAILABLE_ICONS } from '../Icon';
+
+// Auto Docs
+const meta = {
+    title: 'Forms / Text Area',
+    component: TextArea,
+
+    // Display Properties
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'A component that is used to get user input in a text area field.',
+        },
+    },
+
+    // Component-level argTypes
+    argTypes: {
+        label: {
+            description: 'Label for the TextArea.',
+            table: {
+                defaultValue: { summary: textAreaDefaults.label },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        placeholder: {
+            description: 'Placeholder for the Text Area.',
+            table: {
+                defaultValue: { summary: textAreaDefaults.placeholder },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        icon: {
+            description: 'The icon to display in the Text Area.',
+            type: 'string',
+            options: AVAILABLE_ICONS,
+            table: {
+                defaultValue: { summary: 'undefined' },
+            },
+            control: {
+                type: 'select',
+            },
+        },
+        error: {
+            description: 'Enforce error state on the TextArea.',
+            table: {
+                defaultValue: { summary: textAreaDefaults.error },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        warning: {
+            description: 'Enforce warning state on the TextArea.',
+            table: {
+                defaultValue: { summary: textAreaDefaults.warning },
+            },
+            control: {
+                type: 'text',
+            },
+        },
+        isSuccess: {
+            description: 'Enforce success state on the TextArea.',
+            table: {
+                defaultValue: { summary: textAreaDefaults?.isSuccess?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isDisabled: {
+            description: 'Enforce disabled state on the TextArea.',
+            table: {
+                defaultValue: { summary: textAreaDefaults?.isDisabled?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isInvalid: {
+            description: 'Enforce invalid state on the TextArea.',
+            table: {
+                defaultValue: { summary: textAreaDefaults?.isInvalid?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isReadOnly: {
+            description: 'Enforce read only state on the TextArea.',
+            table: {
+                defaultValue: { summary: textAreaDefaults?.isReadOnly?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+        isRequired: {
+            description: 'Enforce required state on the TextArea.',
+            table: {
+                defaultValue: { summary: textAreaDefaults?.isRequired?.toString() },
+            },
+            control: {
+                type: 'boolean',
+            },
+        },
+    },
+
+    // Define defaults
+    args: {
+        label: textAreaDefaults.label,
+        placeholder: textAreaDefaults.placeholder,
+        icon: textAreaDefaults.icon,
+        error: textAreaDefaults.error,
+        warning: textAreaDefaults.warning,
+        isSuccess: textAreaDefaults.isSuccess,
+        isDisabled: textAreaDefaults.isDisabled,
+        isInvalid: textAreaDefaults.isInvalid,
+        isReadOnly: textAreaDefaults.isReadOnly,
+        isRequired: textAreaDefaults.isRequired,
+    },
+} satisfies Meta<typeof TextArea>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+    render: (props) => <TextArea {...props} />,
+};
+
+export const status = () => (
+    <GridList isVertical>
+        <TextArea label="Success Status" isSuccess />
+        <TextArea label="Error Status" isInvalid error="Error Message" />
+        <TextArea label="Warning Status" warning="Warning Message" />
+    </GridList>
+);
+
+export const states = () => (
+    <GridList isVertical>
+        <TextArea label="Disabled TextArea" isDisabled />
+        <TextArea label="Readonly TextArea" isReadOnly />
+        <TextArea label="Required TextArea" isRequired />
+    </GridList>
+);
diff --git a/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.tsx b/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.tsx
new file mode 100644
index 0000000000000..bf7de3520f9e5
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.tsx
@@ -0,0 +1,80 @@
+import React from 'react';
+
+import { TextAreaProps } from './types';
+
+import {
+    ErrorMessage,
+    Label,
+    Required,
+    StyledIcon,
+    StyledStatusIcon,
+    TextAreaContainer,
+    TextAreaField,
+    TextAreaWrapper,
+    WarningMessage,
+} from './components';
+
+export const textAreaDefaults: TextAreaProps = {
+    label: 'Label',
+    placeholder: 'Placeholder',
+    error: '',
+    warning: '',
+    isSuccess: false,
+    isDisabled: false,
+    isInvalid: false,
+    isReadOnly: false,
+    isRequired: false,
+};
+
+export const TextArea = ({
+    label = textAreaDefaults.label,
+    placeholder = textAreaDefaults.placeholder,
+    icon, // default undefined
+    error = textAreaDefaults.error,
+    warning = textAreaDefaults.warning,
+    isSuccess = textAreaDefaults.isSuccess,
+    isDisabled = textAreaDefaults.isDisabled,
+    isInvalid = textAreaDefaults.isInvalid,
+    isReadOnly = textAreaDefaults.isReadOnly,
+    isRequired = textAreaDefaults.isRequired,
+    ...props
+}: TextAreaProps) => {
+    // Invalid state is always true if error is present
+    let invalid = isInvalid;
+    if (error) invalid = true;
+
+    // Input base props
+
+    const textAreaBaseProps = {
+        label,
+        isSuccess,
+        error,
+        warning,
+        isDisabled,
+        isInvalid: invalid,
+    };
+
+    return (
+        <TextAreaWrapper>
+            <Label>
+                {label} {isRequired && <Required>*</Required>}
+            </Label>
+            <TextAreaContainer {...textAreaBaseProps}>
+                {icon && <StyledIcon icon={icon} size="lg" />}
+                <TextAreaField
+                    icon={icon}
+                    placeholder={placeholder}
+                    readOnly={isReadOnly}
+                    disabled={isDisabled}
+                    required={isRequired}
+                    {...props}
+                />
+                {isSuccess && <StyledStatusIcon icon="CheckCircle" color="green" size="lg" />}
+                {invalid && <StyledStatusIcon icon="WarningAmber" color="red" size="lg" />}
+                {warning && <StyledStatusIcon icon="ErrorOutline" color="yellow" size="lg" />}
+            </TextAreaContainer>
+            {invalid && error && <ErrorMessage>{error}</ErrorMessage>}
+            {warning && <WarningMessage>{warning}</WarningMessage>}
+        </TextAreaWrapper>
+    );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/TextArea/components.ts b/datahub-web-react/src/alchemy-components/components/TextArea/components.ts
new file mode 100644
index 0000000000000..dd208d3b232f1
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/TextArea/components.ts
@@ -0,0 +1,106 @@
+import styled from 'styled-components';
+
+import theme, { colors, radius, borders, spacing, typography, sizes } from '@components/theme';
+import { getStatusColors } from '@components/theme/utils';
+
+import { Icon, IconNames } from '../Icon';
+
+import { formLabelTextStyles, inputValueTextStyles, inputPlaceholderTextStyles } from '../commonStyles';
+
+import { TextAreaProps } from './types';
+
+const minHeight = '100px';
+
+const defaultFlexStyles = {
+    display: 'flex',
+    justifyContent: 'space-between',
+    alignItems: 'flex-start',
+};
+
+const defaultMessageStyles = {
+    marginTop: spacing.xxsm,
+    fontSize: typography.fontSizes.sm,
+};
+
+export const TextAreaWrapper = styled.div({
+    ...defaultFlexStyles,
+    flexDirection: 'column',
+    width: '100%',
+});
+
+export const StyledIcon = styled(Icon)({
+    minWidth: '16px',
+    paddingLeft: spacing.sm,
+    marginTop: spacing.sm,
+});
+
+export const TextAreaContainer = styled.div(
+    ({ isSuccess, warning, isDisabled, isInvalid }: TextAreaProps) => ({
+        border: `${borders['1px']} ${getStatusColors(isSuccess, warning, isInvalid)}`,
+        backgroundColor: isDisabled ? colors.gray[100] : colors.white,
+    }),
+    {
+        ...defaultFlexStyles,
+        position: 'relative',
+        minWidth: '160px',
+        minHeight,
+        width: sizes.full,
+        borderRadius: radius.md,
+        flex: 1,
+        color: colors.gray[400], // first icon color
+
+        '&:focus-within': {
+            borderColor: colors.violet[200],
+            outline: `${borders['1px']} ${colors.violet[200]}`,
+        },
+    },
+);
+
+export const TextAreaField = styled.textarea<{ icon?: IconNames }>(({ icon }) => ({
+    padding: `${spacing.sm} ${spacing.md}`,
+    borderRadius: radius.md,
+    border: borders.none,
+    width: '100%',
+    minHeight,
+
+    ...inputValueTextStyles(),
+
+    // Account for icon placement
+    ...(icon && {
+        paddingLeft: spacing.xsm,
+    }),
+
+    '&:focus': {
+        outline: 'none',
+    },
+
+    '&::placeholder': {
+        ...inputPlaceholderTextStyles,
+    },
+}));
+
+export const Label = styled.div({
+    ...formLabelTextStyles,
+    marginBottom: spacing.xxsm,
+    textAlign: 'left',
+});
+
+export const Required = styled.span({
+    color: colors.red[500],
+});
+
+export const ErrorMessage = styled.div({
+    ...defaultMessageStyles,
+    color: theme.semanticTokens.colors.error,
+});
+
+export const WarningMessage = styled.div({
+    ...defaultMessageStyles,
+    color: theme.semanticTokens.colors.warning,
+});
+
+export const StyledStatusIcon = styled(Icon)({
+    position: 'absolute',
+    top: spacing.sm,
+    right: spacing.sm,
+});
diff --git a/datahub-web-react/src/alchemy-components/components/TextArea/index.ts b/datahub-web-react/src/alchemy-components/components/TextArea/index.ts
new file mode 100644
index 0000000000000..f367b1a5851a4
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/TextArea/index.ts
@@ -0,0 +1,2 @@
+export { TextArea, textAreaDefaults } from './TextArea';
+export type { TextAreaProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/TextArea/types.ts b/datahub-web-react/src/alchemy-components/components/TextArea/types.ts
new file mode 100644
index 0000000000000..a6edd05e8bd09
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/TextArea/types.ts
@@ -0,0 +1,15 @@
+import { TextareaHTMLAttributes } from 'react';
+import { IconNames } from '../Icon';
+
+export interface TextAreaProps extends TextareaHTMLAttributes<HTMLTextAreaElement> {
+    label: string;
+    placeholder?: string;
+    icon?: IconNames;
+    error?: string;
+    warning?: string;
+    isSuccess?: boolean;
+    isDisabled?: boolean;
+    isInvalid?: boolean;
+    isReadOnly?: boolean;
+    isRequired?: boolean;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Tooltip/Tooltip.tsx b/datahub-web-react/src/alchemy-components/components/Tooltip/Tooltip.tsx
new file mode 100644
index 0000000000000..03a2da4382c28
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Tooltip/Tooltip.tsx
@@ -0,0 +1,6 @@
+import { Tooltip, TooltipProps } from 'antd';
+import * as React from 'react';
+
+export default function DataHubTooltip(props: TooltipProps & React.RefAttributes<unknown>) {
+    return <Tooltip {...props} showArrow={false} />;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/Tooltip/index.ts b/datahub-web-react/src/alchemy-components/components/Tooltip/index.ts
new file mode 100644
index 0000000000000..49ab72ce02386
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Tooltip/index.ts
@@ -0,0 +1 @@
+export { default as Tooltip } from './Tooltip';
diff --git a/datahub-web-react/src/alchemy-components/components/commonStyles.ts b/datahub-web-react/src/alchemy-components/components/commonStyles.ts
new file mode 100644
index 0000000000000..04069f5950bc4
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/commonStyles.ts
@@ -0,0 +1,23 @@
+import { colors, typography } from '@components/theme';
+
+export const INPUT_MAX_HEIGHT = '40px';
+
+export const formLabelTextStyles = {
+    fontWeight: typography.fontWeights.normal,
+    fontSize: typography.fontSizes.md,
+    color: colors.gray[600],
+};
+
+export const inputValueTextStyles = (size = 'md') => ({
+    fontFamily: typography.fonts.body,
+    fontWeight: typography.fontWeights.normal,
+    fontSize: typography.fontSizes[size],
+    color: colors.gray[700],
+});
+
+export const inputPlaceholderTextStyles = {
+    fontFamily: typography.fonts.body,
+    fontWeight: typography.fontWeights.normal,
+    fontSize: typography.fontSizes.md,
+    color: colors.gray[400],
+};
diff --git a/datahub-web-react/src/alchemy-components/components/dataviz/utils.ts b/datahub-web-react/src/alchemy-components/components/dataviz/utils.ts
new file mode 100644
index 0000000000000..15e2783d2905d
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/dataviz/utils.ts
@@ -0,0 +1,11 @@
+// Number Abbreviations
+export const abbreviateNumber = (str) => {
+    const number = parseFloat(str);
+    if (Number.isNaN(number)) return str;
+    if (number < 1000) return number;
+    const abbreviations = ['K', 'M', 'B', 'T'];
+    const index = Math.floor(Math.log10(number) / 3);
+    const suffix = abbreviations[index - 1];
+    const shortNumber = number / 10 ** (index * 3);
+    return `${shortNumber}${suffix}`;
+};
diff --git a/datahub-web-react/src/alchemy-components/index.ts b/datahub-web-react/src/alchemy-components/index.ts
new file mode 100644
index 0000000000000..7e40d343e884d
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/index.ts
@@ -0,0 +1,24 @@
+// example usage: import { colors } from '@components/theme';
+export * from './theme';
+
+// example usage: import { Button } from '@components';
+export * from './components/Avatar';
+export * from './components/Badge';
+export * from './components/BarChart';
+export * from './components/Button';
+export * from './components/Card';
+export * from './components/Checkbox';
+export * from './components/Heading';
+export * from './components/Icon';
+export * from './components/Input';
+export * from './components/LineChart';
+export * from './components/PageTitle';
+export * from './components/Pills';
+export * from './components/Popover';
+export * from './components/SearchBar';
+export * from './components/Select';
+export * from './components/Switch';
+export * from './components/Table';
+export * from './components/Text';
+export * from './components/TextArea';
+export * from './components/Tooltip';
diff --git a/datahub-web-react/src/alchemy-components/theme/config/constants.ts b/datahub-web-react/src/alchemy-components/theme/config/constants.ts
new file mode 100644
index 0000000000000..84e55c78f85be
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/config/constants.ts
@@ -0,0 +1 @@
+export const DEFAULT_VALUE = 500;
diff --git a/datahub-web-react/src/alchemy-components/theme/config/index.ts b/datahub-web-react/src/alchemy-components/theme/config/index.ts
new file mode 100644
index 0000000000000..4fbce9564d5fc
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/config/index.ts
@@ -0,0 +1,2 @@
+export * from './constants';
+export * from './types';
diff --git a/datahub-web-react/src/alchemy-components/theme/config/types.ts b/datahub-web-react/src/alchemy-components/theme/config/types.ts
new file mode 100644
index 0000000000000..79ba2e27018f7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/config/types.ts
@@ -0,0 +1,47 @@
+// General types
+export type SizeOptions = 'xs' | 'sm' | 'md' | 'lg' | 'xl';
+
+// Color types
+export interface Color {
+    100: string;
+    200: string;
+    300: string;
+    400: string;
+    500: string;
+    600: string;
+    700: string;
+    800: string;
+    900: string;
+}
+export type ColorOptions = 'white' | 'black' | 'violet' | 'green' | 'red' | 'blue' | 'gray' | 'yellow';
+export type MiscColorOptions = 'transparent' | 'current' | 'inherit';
+export type ColorShadeOptions = 'light' | 'default' | 'dark';
+
+// Typography types
+export type FontSizeOptions = 'xs' | 'sm' | 'md' | 'lg' | 'xl' | '2xl' | '3xl' | '4xl';
+export type FontWeightOptions = 'normal' | 'medium' | 'semiBold' | 'bold';
+export type FontColorOptions = MiscColorOptions | ColorOptions;
+
+// Border radius types
+export type BorderRadiusOptions = 'none' | 'sm' | 'md' | 'lg' | 'xl' | 'full';
+
+// Box shadow types
+export type BoxShadowOptions = 'xs' | 'sm' | 'md' | 'lg' | 'xl' | '2xl' | 'inner' | 'outline' | 'dropdown' | 'none';
+
+// Spacing types
+export type SpacingOptions = 'none' | 'sm' | 'md' | 'lg' | 'xl' | '3xl' | '4xl';
+
+// Transform types
+export type RotationOptions = '0' | '90' | '180' | '270';
+
+// Variant types
+export type VariantOptions = 'filled' | 'outline';
+
+// Alignment types
+export type AlignmentOptions = 'left' | 'right' | 'center' | 'justify';
+
+// Avatar Size options
+export type AvatarSizeOptions = 'sm' | 'md' | 'lg' | 'default';
+
+// Icon Alignment types
+export type IconAlignmentOptions = 'horizontal' | 'vertical';
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/blur.ts b/datahub-web-react/src/alchemy-components/theme/foundations/blur.ts
new file mode 100644
index 0000000000000..88f117f907679
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/blur.ts
@@ -0,0 +1,12 @@
+const blur = {
+    none: 0,
+    sm: '4px',
+    base: '8px',
+    md: '12px',
+    lg: '16px',
+    xl: '24px',
+    '2xl': '40px',
+    '3xl': '64px',
+};
+
+export default blur;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/borders.ts b/datahub-web-react/src/alchemy-components/theme/foundations/borders.ts
new file mode 100644
index 0000000000000..202fd1f8c3e4f
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/borders.ts
@@ -0,0 +1,9 @@
+const borders = {
+    none: 0,
+    '1px': '1px solid',
+    '2px': '2px solid',
+    '4px': '4px solid',
+    '8px': '8px solid',
+};
+
+export default borders;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/breakpoints.ts b/datahub-web-react/src/alchemy-components/theme/foundations/breakpoints.ts
new file mode 100644
index 0000000000000..722cc30ee3fd5
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/breakpoints.ts
@@ -0,0 +1,10 @@
+const breakpoints = {
+    base: '0em',
+    sm: '30em',
+    md: '48em',
+    lg: '62em',
+    xl: '80em',
+    '2xl': '96em',
+};
+
+export default breakpoints;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/colors.ts b/datahub-web-react/src/alchemy-components/theme/foundations/colors.ts
new file mode 100644
index 0000000000000..760127a8b0f78
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/colors.ts
@@ -0,0 +1,98 @@
+const colors = {
+    transparent: 'transparent',
+    current: 'currentColor',
+    white: '#ffffff',
+    black: '#000000',
+
+    gray: {
+        100: '#EBECF0',
+        200: '#CFD1DA',
+        300: '#A9ADBD',
+        400: '#81879F',
+        500: '#5B6282', // primary value
+        600: '#374066',
+        700: '#2F3657',
+        800: '#272D48',
+        900: '#231A58',
+        1000: '#F1F3FD',
+        1100: '#F1FBFE',
+        1200: '#FBF3EF',
+        1300: '#F7FBF4',
+        1400: '#E9EAEE',
+        1500: '#F9FAFC',
+        1600: '#F5F6FA',
+        1700: '#5F6685',
+        1800: '#8088A3',
+    },
+
+    violet: {
+        100: '#CAC3F1',
+        200: '#B0A7EA',
+        300: '#8C7EE0',
+        400: '#7565DA',
+        500: '#533FD1', // primary value
+        600: '#4C39BE',
+        700: '#3B2D94',
+        800: '#2E2373',
+        900: '#231A58',
+        1000: '#E5E2F8',
+    },
+
+    green: {
+        100: '#d5e9c9',
+        200: '#c0deaf',
+        300: '#a4cf8a',
+        400: '#92c573',
+        500: '#77b750', // primary value
+        600: '#6ca749',
+        700: '#548239',
+        800: '#41652c',
+        900: '#324d22',
+        1000: '#0D7543',
+        1100: '#E1F0D6',
+    },
+
+    red: {
+        100: '#f6d5d5',
+        200: '#f2c1c1',
+        300: '#eca5a5',
+        400: '#e99393',
+        500: '#e37878', // primary value
+        600: '#cf6d6d',
+        700: '#a15555',
+        800: '#7d4242',
+        900: '#5f3232',
+        1000: '#C4360B',
+        1100: '#F3DACE',
+    },
+
+    blue: {
+        100: '#eff8fc',
+        200: '#e6f5fb',
+        300: '#ccebf6',
+        400: '#5abde1', // primary value
+        500: '#51aacb',
+        600: '#4897b4',
+        700: '#448ea9',
+        800: '#367187',
+        900: '#285565',
+        1000: '#09739A',
+        1100: '#CCEBF6',
+    },
+
+    yellow: {
+        100: '#fcedc7',
+        200: '#fae4ab',
+        300: '#f8d785',
+        400: '#f6d06d',
+        500: '#eeae09', // primary value
+        600: '#deb242',
+        700: '#ad8b34',
+        800: '#866c28',
+        900: '#66521f',
+        1000: '#C77100',
+        1100: '#FCEDC7',
+    },
+};
+
+export default colors;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/index.ts b/datahub-web-react/src/alchemy-components/theme/foundations/index.ts
new file mode 100644
index 0000000000000..e4c81d77f0755
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/index.ts
@@ -0,0 +1,27 @@
+import blur from './blur';
+import borders from './borders';
+import breakpoints from './breakpoints';
+import colors from './colors';
+import radius from './radius';
+import shadows from './shadows';
+import sizes from './sizes';
+import spacing from './spacing';
+import transform from './transform';
+import transition from './transition';
+import typography from './typography';
+import zIndices from './zIndex';
+
+export const foundations = {
+    blur,
+    borders,
+    breakpoints,
+    colors,
+    radius,
+    shadows,
+    sizes,
+    spacing,
+    transform,
+    transition,
+    typography,
+    zIndices,
+};
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/radius.ts b/datahub-web-react/src/alchemy-components/theme/foundations/radius.ts
new file mode 100644
index 0000000000000..aa17688a4328a
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/radius.ts
@@ -0,0 +1,9 @@
+const radius = {
+    none: '0',
+    sm: '4px',
+    md: '8px',
+    lg: '12px',
+    full: '100%',
+};
+
+export default radius;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/shadows.ts b/datahub-web-react/src/alchemy-components/theme/foundations/shadows.ts
new file mode 100644
index 0000000000000..e4cc920dd82e5
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/shadows.ts
@@ -0,0 +1,16 @@
+import { BoxShadowOptions } from '../config';
+
+const shadows: Record<BoxShadowOptions, string> = {
+    xs: '0px 1px 2px 0px rgba(0, 0, 0, 0.05)',
+    sm: '0 4px 4px 0 rgba(0 0 0 / 0.25)',
+    md: '0 8px 8px 4px rgba(0 0 0 / 0.25)',
+    lg: '0 12px 12px 8px rgba(0 0 0 / 0.25)',
+    xl: '0 16px 16px 12px rgba(0 0 0 / 0.25)',
+    '2xl': '0 24px 24px 16px rgba(0 0 0 / 0.25)',
+    inner: 'inset 0 2px 4px 0 rgba(0 0 0 / 0.06)',
+    outline: '0 0 0 3px rgba(66, 153, 225, 0.5)',
+    none: 'none',
+    dropdown: '0px 0px 14px 0px #00000026',
+};
+
+export default shadows;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/sizes.ts b/datahub-web-react/src/alchemy-components/theme/foundations/sizes.ts
new file mode 100644
index 0000000000000..c1050e328b109
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/sizes.ts
@@ -0,0 +1,7 @@
+const sizes = {
+    max: 'max-content',
+    min: 'min-content',
+    full: '100%',
+};
+
+export default sizes;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/spacing.ts b/datahub-web-react/src/alchemy-components/theme/foundations/spacing.ts
new file mode 100644
index 0000000000000..8adb1bd176901
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/spacing.ts
@@ -0,0 +1,12 @@
+const spacing = {
+    none: '0px',
+    xxsm: '4px',
+    xsm: '8px',
+    sm: '12px',
+    md: '16px',
+    lg: '24px',
+    xlg: '32px',
+    xxlg: '48px',
+};
+
+export default spacing;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/transform.ts b/datahub-web-react/src/alchemy-components/theme/foundations/transform.ts
new file mode 100644
index 0000000000000..1065250dc191e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/transform.ts
@@ -0,0 +1,10 @@
+const transform = {
+    rotate: {
+        none: 'rotate(0deg)',
+        90: 'rotate(90deg)',
+        180: 'rotate(180deg)',
+        270: 'rotate(270deg)',
+    },
+};
+
+export default transform;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/transition.ts b/datahub-web-react/src/alchemy-components/theme/foundations/transition.ts
new file mode 100644
index 0000000000000..83c541b9e52f0
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/transition.ts
@@ -0,0 +1,32 @@
+const transitionProperty = {
+    common: `background-color, border-color, color, fill,
+	stroke, opacity, box-shadow, transform`,
+    colors: 'background-color, border-color, color, fill, stroke',
+    dimensions: 'width, height',
+    position: 'left, right, top, bottom',
+    background: 'background-color, background-image, background-position',
+};
+
+const transitionTimingFunction = {
+    'ease-in': 'cubic-bezier(0.4, 0, 1, 1)',
+    'ease-out': 'cubic-bezier(0, 0, 0.2, 1)',
+    'ease-in-out': 'cubic-bezier(0.4, 0, 0.2, 1)',
+};
+
+const transitionDuration = {
+    'ultra-fast': '50ms',
+    faster: '100ms',
+    fast: '150ms',
+    normal: '200ms',
+    slow: '300ms',
+    slower: '400ms',
+    'ultra-slow': '500ms',
+};
+
+const transition = {
+    property: transitionProperty,
+    easing: transitionTimingFunction,
+    duration: transitionDuration,
+};
+
+export default transition;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/typography.ts b/datahub-web-react/src/alchemy-components/theme/foundations/typography.ts
new file mode 100644
index 0000000000000..3b63e678558e4
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/typography.ts
@@ -0,0 +1,52 @@
+const typography = {
+    letterSpacings: {
+        tighter: '-2px',
+        tight: '-1px',
+        normal: '0',
+        wide: '1px',
+        wider: '2px',
+        widest: '4px',
+    },
+
+    lineHeights: {
+        normal: 'normal',
+        none: 1,
+        xs: '16px',
+        sm: '20px',
+        md: '24px',
+        lg: '28px',
+        xl: '32px',
+        '2xl': '36px',
+        '3xl': '40px',
+        '4xl': '44px',
+    },
+
+    fontWeights: {
+        normal: 400, // regular
+        medium: 500,
+        semiBold: 600,
+        bold: 700,
+    },
+
+    fonts: {
+        heading: `'Mulish', -apple-system, BlinkMacSystemFont,
+		'Segoe UI', Helvetica, Arial, sans-serif`,
+        body: `'Mulish', -apple-system, BlinkMacSystemFont,
+		'Segoe UI', Helvetica, Arial, sans-serif`,
+        mono: `SFMono-Regular, Menlo, Monaco, Consolas,
+		'Liberation Mono', 'Courier New', monospace`,
+    },
+
+    fontSizes: {
+        xs: '10px',
+        sm: '12px',
+        md: '14px', // default body text size
+        lg: '16px',
+        xl: '18px',
+        '2xl': '20px',
+        '3xl': '22px',
+        '4xl': '24px',
+    },
+};
+
+export default typography;
diff --git a/datahub-web-react/src/alchemy-components/theme/foundations/zIndex.ts b/datahub-web-react/src/alchemy-components/theme/foundations/zIndex.ts
new file mode 100644
index 0000000000000..318783d77e4e5
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/foundations/zIndex.ts
@@ -0,0 +1,17 @@
+const zIndices = {
+    hide: -1,
+    auto: 'auto',
+    base: 0,
+    docked: 10,
+    dropdown: 1000,
+    sticky: 1100,
+    banner: 1200,
+    overlay: 1300,
+    modal: 1400,
+    popover: 1500,
+    skipLink: 1600,
+    toast: 1700,
+    tooltip: 1800,
+};
+
+export default zIndices;
diff --git a/datahub-web-react/src/alchemy-components/theme/index.ts b/datahub-web-react/src/alchemy-components/theme/index.ts
new file mode 100644
index 0000000000000..65bad35f827cc
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/index.ts
@@ -0,0 +1,30 @@
+import * as config from './config';
+import * as utils from './utils';
+
+import { foundations } from './foundations';
+import { semanticTokens } from './semantic-tokens';
+
+const theme = {
+    semanticTokens,
+    ...foundations,
+    config,
+    utils,
+};
+
+export const {
+    colors,
+    spacing,
+    radius,
+    shadows,
+    typography,
+    breakpoints,
+    zIndices,
+    transform,
+    transition,
+    sizes,
+    borders,
+    blur,
+} = theme;
+
+export type Theme = typeof theme;
+export default theme;
diff --git a/datahub-web-react/src/alchemy-components/theme/semantic-tokens.ts b/datahub-web-react/src/alchemy-components/theme/semantic-tokens.ts
new file mode 100644
index 0000000000000..29bd20d11f854
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/semantic-tokens.ts
@@ -0,0 +1,21 @@
+import { foundations } from './foundations';
+
+const { colors } = foundations;
+
+export const semanticTokens = {
+    colors: {
+        'body-text': colors.gray[800],
+        'body-bg': colors.white,
+        'border-color': colors.gray[200],
+        'inverse-text': colors.white,
+        'subtle-bg': colors.gray[100],
+        'subtle-text': colors.gray[600],
+        'placeholder-color': colors.gray[500],
+        primary: colors.violet[500],
+        secondary: colors.blue[500],
+        error: colors.red[500],
+        success: colors.green[500],
+        warning: colors.yellow[500],
+        info: colors.blue[500],
+    },
+};
diff --git a/datahub-web-react/src/alchemy-components/theme/utils.ts b/datahub-web-react/src/alchemy-components/theme/utils.ts
new file mode 100644
index 0000000000000..4c20ca39f06ee
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/theme/utils.ts
@@ -0,0 +1,62 @@
+/*
+	Theme Utils that can be used anywhere in the app
+*/
+
+import { FontSizeOptions, ColorOptions, MiscColorOptions, RotationOptions, DEFAULT_VALUE } from './config';
+import { foundations } from './foundations';
+import { semanticTokens } from './semantic-tokens';
+
+const { colors, typography, transform } = foundations;
+/*
+	Get the color value for a given color
+	Falls back to `color.black` if the color is not found
+	@param color - the color to get the value for
+*/
+export const getColor = (color?: MiscColorOptions | ColorOptions, value: number | string = DEFAULT_VALUE) => {
+    if (!color) return colors.black;
+    if (color === 'inherit' || color === 'transparent' || color === 'current') return colors;
+    if (color === 'white') return colors.white;
+    if (color === 'black') return colors.black;
+    const colorValue = colors[color];
+    if (!colorValue) return colors.black;
+    return colors[color][value];
+};
+
+/*
+	Get the font size value for a given size
+	@param size - the size of the font
+*/
+export const getFontSize = (size?: FontSizeOptions) => {
+    let sizeValue = size || '';
+    if (!size) sizeValue = 'md';
+    return typography.fontSizes[sizeValue];
+};
+
+/*
+	Get the rotation transform value for a given rotation
+	@param r - the rotation to get the transform value for
+*/
+export const getRotationTransform = (rotate?: RotationOptions) => {
+    if (!rotate) return '';
+    return transform.rotate[rotate || '0'];
+};
+
+/**
+ * Get the status color depending on the flags that are true
+ * @param {string} [error] - Error definition, if any.
+ * @param {boolean} [isSuccess] - Boolean flag indicating success.
+ * @param {string} [warning] - Warning definition, if any.
+ * @returns {string} - The status color based on the provided flags.
+ */
+export const getStatusColors = (isSuccess?: boolean, warning?: string, isInvalid?: boolean): string => {
+    if (isInvalid) {
+        return colors.red[600];
+    }
+    if (isSuccess) {
+        return colors.green[600];
+    }
+    if (warning) {
+        return colors.yellow[600];
+    }
+    return semanticTokens.colors['border-color'];
+};
diff --git a/datahub-web-react/src/app/AdminConsole.tsx b/datahub-web-react/src/app/AdminConsole.tsx
index f6395a3bd3cb8..a79575c3d88d7 100644
--- a/datahub-web-react/src/app/AdminConsole.tsx
+++ b/datahub-web-react/src/app/AdminConsole.tsx
@@ -37,8 +37,8 @@ export const AdminConsole = (): JSX.Element => {
     const [adminConsoleOpen, setAdminConsoleOpen] = useState(false);
     const { config } = useAppConfig();
 
-    const isAnalyticsEnabled = config?.analyticsConfig.enabled;
-    const isPoliciesEnabled = config?.policiesConfig.enabled;
+    const isAnalyticsEnabled = config?.analyticsConfig?.enabled;
+    const isPoliciesEnabled = config?.policiesConfig?.enabled;
 
     const showAnalytics = (isAnalyticsEnabled && me && me?.platformPrivileges?.viewAnalytics) || false;
     const showPolicyBuilder = (isPoliciesEnabled && me && me?.platformPrivileges?.managePolicies) || false;
diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx
index 3343260c72bcf..024b6e0add15a 100644
--- a/datahub-web-react/src/app/SearchRoutes.tsx
+++ b/datahub-web-react/src/app/SearchRoutes.tsx
@@ -11,23 +11,36 @@ import { AnalyticsPage } from './analyticsDashboard/components/AnalyticsPage';
 import { ManageIngestionPage } from './ingest/ManageIngestionPage';
 import GlossaryRoutes from './glossary/GlossaryRoutes';
 import { SettingsPage } from './settings/SettingsPage';
+import { useUserContext } from './context/useUserContext';
 import DomainRoutes from './domain/DomainRoutes';
-import { useBusinessAttributesFlag, useIsAppConfigContextLoaded, useIsNestedDomainsEnabled } from './useAppConfig';
+import {
+    useAppConfig,
+    useBusinessAttributesFlag,
+    useIsAppConfigContextLoaded,
+    useIsNestedDomainsEnabled,
+} from './useAppConfig';
 import { ManageDomainsPage } from './domain/ManageDomainsPage';
 import { BusinessAttributes } from './businessAttribute/BusinessAttributes';
+import StructuredProperties from './govern/structuredProperties/StructuredProperties';
 /**
  * Container for all searchable page routes
  */
 export const SearchRoutes = (): JSX.Element => {
     const entityRegistry = useEntityRegistry();
+    const me = useUserContext();
     const isNestedDomainsEnabled = useIsNestedDomainsEnabled();
     const entities = isNestedDomainsEnabled
         ? entityRegistry.getEntitiesForSearchRoutes()
         : entityRegistry.getNonGlossaryEntities();
+    const { config } = useAppConfig();
 
     const businessAttributesFlag = useBusinessAttributesFlag();
     const appConfigContextLoaded = useIsAppConfigContextLoaded();
 
+    const showStructuredProperties =
+        config?.featureFlags?.showManageStructuredProperties &&
+        (me.platformPrivileges?.manageStructuredProperties || me.platformPrivileges?.viewStructuredPropertiesPage);
+
     return (
         <SearchablePage>
             <Switch>
@@ -53,6 +66,9 @@ export const SearchRoutes = (): JSX.Element => {
                 <Route path={PageRoutes.INGESTION} render={() => <ManageIngestionPage />} />
                 <Route path={PageRoutes.SETTINGS} render={() => <SettingsPage />} />
                 <Route path={`${PageRoutes.GLOSSARY}*`} render={() => <GlossaryRoutes />} />
+                {showStructuredProperties && (
+                    <Route path={PageRoutes.STRUCTURED_PROPERTIES} render={() => <StructuredProperties />} />
+                )}
                 <Route
                     path={PageRoutes.BUSINESS_ATTRIBUTE}
                     render={() => {
diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts
index c3a57830b8c50..9152f2fb8eedb 100644
--- a/datahub-web-react/src/app/analytics/event.ts
+++ b/datahub-web-react/src/app/analytics/event.ts
@@ -1,4 +1,12 @@
-import { DataHubViewType, EntityType, RecommendationRenderType, ScenarioType } from '../../types.generated';
+import {
+    AllowedValue,
+    DataHubViewType,
+    EntityType,
+    PropertyCardinality,
+    PropertyValueInput,
+    RecommendationRenderType,
+    ScenarioType,
+} from '../../types.generated';
 import { EmbedLookupNotFoundReason } from '../embed/lookup/constants';
 import { Direction } from '../lineage/types';
 import { FilterMode } from '../search/utils/constants';
@@ -82,6 +90,14 @@ export enum EventType {
     EmbedProfileViewInDataHubEvent,
     EmbedLookupNotFoundEvent,
     CreateBusinessAttributeEvent,
+    CreateStructuredPropertyClickEvent,
+    CreateStructuredPropertyEvent,
+    EditStructuredPropertyEvent,
+    DeleteStructuredPropertyEvent,
+    ViewStructuredPropertyEvent,
+    ApplyStructuredPropertyEvent,
+    UpdateStructuredPropertyOnAssetEvent,
+    RemoveStructuredPropertyEvent,
 }
 
 /**
@@ -640,6 +656,64 @@ export interface CreateBusinessAttributeEvent extends BaseEvent {
     name: string;
 }
 
+export interface CreateStructuredPropertyClickEvent extends BaseEvent {
+    type: EventType.CreateStructuredPropertyClickEvent;
+}
+
+interface StructuredPropertyEvent extends BaseEvent {
+    propertyType: string;
+    appliesTo: string[];
+    qualifiedName?: string;
+    allowedAssetTypes?: string[];
+    allowedValues?: AllowedValue[];
+    cardinality?: PropertyCardinality;
+    showInFilters?: boolean;
+    isHidden: boolean;
+    showInSearchFilters: boolean;
+    showAsAssetBadge: boolean;
+    showInAssetSummary: boolean;
+    showInColumnsTable: boolean;
+}
+
+export interface CreateStructuredPropertyEvent extends StructuredPropertyEvent {
+    type: EventType.CreateStructuredPropertyEvent;
+}
+
+export interface EditStructuredPropertyEvent extends StructuredPropertyEvent {
+    type: EventType.EditStructuredPropertyEvent;
+    propertyUrn: string;
+}
+
+export interface DeleteStructuredPropertyEvent extends StructuredPropertyEvent {
+    type: EventType.DeleteStructuredPropertyEvent;
+    propertyUrn: string;
+}
+
+export interface ViewStructuredPropertyEvent extends BaseEvent {
+    type: EventType.ViewStructuredPropertyEvent;
+    propertyUrn: string;
+}
+
+interface StructuredPropertyOnAssetEvent extends BaseEvent {
+    propertyUrn: string;
+    propertyType: string;
+    assetUrn: string;
+    assetType: EntityType;
+}
+export interface ApplyStructuredPropertyEvent extends StructuredPropertyOnAssetEvent {
+    type: EventType.ApplyStructuredPropertyEvent;
+    values: PropertyValueInput[];
+}
+
+export interface UpdateStructuredPropertyOnAssetEvent extends StructuredPropertyOnAssetEvent {
+    type: EventType.UpdateStructuredPropertyOnAssetEvent;
+    values: PropertyValueInput[];
+}
+
+export interface RemoveStructuredPropertyEvent extends StructuredPropertyOnAssetEvent {
+    type: EventType.RemoveStructuredPropertyEvent;
+}
+
 /**
  * Event consisting of a union of specific event types.
  */
@@ -718,4 +792,12 @@ export type Event =
     | EmbedProfileViewEvent
     | EmbedProfileViewInDataHubEvent
     | EmbedLookupNotFoundEvent
-    | CreateBusinessAttributeEvent;
+    | CreateBusinessAttributeEvent
+    | CreateStructuredPropertyClickEvent
+    | CreateStructuredPropertyEvent
+    | EditStructuredPropertyEvent
+    | DeleteStructuredPropertyEvent
+    | ViewStructuredPropertyEvent
+    | ApplyStructuredPropertyEvent
+    | UpdateStructuredPropertyOnAssetEvent
+    | RemoveStructuredPropertyEvent;
diff --git a/datahub-web-react/src/app/analyticsDashboard/components/AnalyticsPage.tsx b/datahub-web-react/src/app/analyticsDashboard/components/AnalyticsPage.tsx
index bf0e5af2427bb..f139ef7368a74 100644
--- a/datahub-web-react/src/app/analyticsDashboard/components/AnalyticsPage.tsx
+++ b/datahub-web-react/src/app/analyticsDashboard/components/AnalyticsPage.tsx
@@ -121,11 +121,11 @@ export const AnalyticsPage = () => {
                                 placeholder="Select a domain"
                                 onChange={onDomainChange}
                                 filterOption={(input, option) =>
-                                    option?.children.toLowerCase().indexOf(input.toLowerCase()) >= 0
+                                    option?.children?.toLowerCase()?.indexOf(input.toLowerCase()) >= 0
                                 }
                             >
                                 <Select.Option value="ALL">All</Select.Option>
-                                {domainData?.listDomains?.domains.map((domainChoice) => (
+                                {domainData?.listDomains?.domains?.map((domainChoice) => (
                                     <Select.Option value={domainChoice.urn}>
                                         {domainChoice?.properties?.name}
                                     </Select.Option>
diff --git a/datahub-web-react/src/app/buildEntityRegistry.ts b/datahub-web-react/src/app/buildEntityRegistry.ts
index 0b70986672be5..181ec7d328a58 100644
--- a/datahub-web-react/src/app/buildEntityRegistry.ts
+++ b/datahub-web-react/src/app/buildEntityRegistry.ts
@@ -24,6 +24,7 @@ import { RoleEntity } from './entity/Access/RoleEntity';
 import { RestrictedEntity } from './entity/restricted/RestrictedEntity';
 import { BusinessAttributeEntity } from './entity/businessAttribute/BusinessAttributeEntity';
 import { SchemaFieldPropertiesEntity } from './entity/schemaField/SchemaFieldPropertiesEntity';
+import { StructuredPropertyEntity } from './entity/structuredProperty/StructuredPropertyEntity';
 
 export default function buildEntityRegistry() {
     const registry = new EntityRegistry();
@@ -52,5 +53,6 @@ export default function buildEntityRegistry() {
     registry.register(new RestrictedEntity());
     registry.register(new BusinessAttributeEntity());
     registry.register(new SchemaFieldPropertiesEntity());
+    registry.register(new StructuredPropertyEntity());
     return registry;
 }
diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx
index bf70bd043fd4a..82977eff4f95b 100644
--- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx
+++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx
@@ -83,7 +83,7 @@ export default function DomainNode({ domain, numDomainChildren, domainUrnToHide,
     const hasDomainChildren = useHasDomainChildren({ domainUrn: domain.urn, numDomainChildren });
 
     const shouldAutoOpen = useMemo(
-        () => !isInSelectMode && entityData?.parentDomains?.domains.some((parent) => parent.urn === domain.urn),
+        () => !isInSelectMode && entityData?.parentDomains?.domains?.some((parent) => parent.urn === domain.urn),
         [isInSelectMode, entityData, domain.urn],
     );
 
diff --git a/datahub-web-react/src/app/embed/lookup/useGetEntityByUrl.ts b/datahub-web-react/src/app/embed/lookup/useGetEntityByUrl.ts
index f11a3a02b4344..49136c39fd242 100644
--- a/datahub-web-react/src/app/embed/lookup/useGetEntityByUrl.ts
+++ b/datahub-web-react/src/app/embed/lookup/useGetEntityByUrl.ts
@@ -31,7 +31,7 @@ const useGetEntityByUrl = (externalUrl: string) => {
     const getLookupData = () => {
         if (!data) return {} as const;
 
-        const entities = data.searchAcrossEntities?.searchResults.map((result) => result.entity) ?? [];
+        const entities = data.searchAcrossEntities?.searchResults?.map((result) => result.entity) ?? [];
         const notFound = entities.length === 0;
         const foundMultiple = entities.length > 1;
         const entity = entities.length === 1 ? entities[0] : null;
diff --git a/datahub-web-react/src/app/entity/Access/RoleEntity.tsx b/datahub-web-react/src/app/entity/Access/RoleEntity.tsx
index ab609b04f104a..58a1ba8dd793b 100644
--- a/datahub-web-react/src/app/entity/Access/RoleEntity.tsx
+++ b/datahub-web-react/src/app/entity/Access/RoleEntity.tsx
@@ -88,4 +88,8 @@ export class RoleEntity implements Entity<Role> {
     supportedCapabilities = () => {
         return new Set([EntityCapabilityType.OWNERS]);
     };
+
+    getGraphName = () => {
+        return 'roleEntity';
+    };
 }
diff --git a/datahub-web-react/src/app/entity/Entity.tsx b/datahub-web-react/src/app/entity/Entity.tsx
index 490f23330c594..c56c97454a1d5 100644
--- a/datahub-web-react/src/app/entity/Entity.tsx
+++ b/datahub-web-react/src/app/entity/Entity.tsx
@@ -172,6 +172,11 @@ export interface Entity<T> {
      */
     getGenericEntityProperties: (data: T) => GenericEntityProperties | null;
 
+    /**
+     * Returns the graph name of the entity, as it appears in the GMS entity registry
+     */
+    getGraphName: () => string;
+
     /**
      * Returns the supported features for the entity
      */
diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx
index 00e7385ff5784..827f0e6692442 100644
--- a/datahub-web-react/src/app/entity/EntityRegistry.tsx
+++ b/datahub-web-react/src/app/entity/EntityRegistry.tsx
@@ -7,6 +7,7 @@ import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from './Enti
 import { GLOSSARY_ENTITY_TYPES } from './shared/constants';
 import { EntitySidebarSection, GenericEntityProperties } from './shared/types';
 import { dictToQueryStringParams, getFineGrainedLineageWithSiblings, urlEncodeUrn } from './shared/utils';
+import PreviewContext from './shared/PreviewContext';
 
 function validatedGet<K, V>(key: K, map: Map<K, V>): V {
     if (map.has(key)) {
@@ -142,13 +143,24 @@ export default class EntityRegistry {
 
     renderPreview<T>(entityType: EntityType, type: PreviewType, data: T): JSX.Element {
         const entity = validatedGet(entityType, this.entityTypeToEntity);
-        return entity.renderPreview(type, data);
+        const genericEntityData = entity.getGenericEntityProperties(data);
+        return (
+            <PreviewContext.Provider value={genericEntityData}>
+                {entity.renderPreview(type, data)}
+            </PreviewContext.Provider>
+        );
     }
 
     renderSearchResult(type: EntityType, searchResult: SearchResult): JSX.Element {
         const entity = validatedGet(type, this.entityTypeToEntity);
+        const genericEntityData = entity.getGenericEntityProperties(searchResult.entity);
+
         return (
-            <SearchResultProvider searchResult={searchResult}>{entity.renderSearch(searchResult)}</SearchResultProvider>
+            <SearchResultProvider searchResult={searchResult}>
+                <PreviewContext.Provider value={genericEntityData}>
+                    {entity.renderSearch(searchResult)}
+                </PreviewContext.Provider>
+            </SearchResultProvider>
         );
     }
 
@@ -205,6 +217,7 @@ export default class EntityRegistry {
                 schemaMetadata: genericEntityProperties?.schemaMetadata,
                 inputFields: genericEntityProperties?.inputFields,
                 canEditLineage: genericEntityProperties?.privileges?.canEditLineage,
+                structuredProperties: genericEntityProperties?.structuredProperties,
             } as FetchedEntity) || undefined
         );
     }
@@ -239,6 +252,10 @@ export default class EntityRegistry {
 
     getCustomCardUrlPath(type: EntityType): string | undefined {
         const entity = validatedGet(type, this.entityTypeToEntity);
-        return entity.getCustomCardUrlPath?.();
+        return entity.getCustomCardUrlPath?.() as string | undefined;
+    }
+
+    getGraphNameFromType(type: EntityType): string {
+        return validatedGet(type, this.entityTypeToEntity).getGraphName();
     }
 }
diff --git a/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx b/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx
index b827a3c37d6a5..442aaf735575a 100644
--- a/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx
+++ b/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx
@@ -59,6 +59,8 @@ export class BusinessAttributeEntity implements Entity<BusinessAttribute> {
 
     getCollectionName = () => 'Business Attributes';
 
+    getGraphName = () => 'businessAttribute';
+
     getCustomCardUrlPath = () => PageRoutes.BUSINESS_ATTRIBUTE;
 
     isBrowseEnabled = () => false;
diff --git a/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeRelatedEntity.tsx b/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeRelatedEntity.tsx
index 46d9d4ea51d24..a73b36d692227 100644
--- a/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeRelatedEntity.tsx
+++ b/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeRelatedEntity.tsx
@@ -18,7 +18,7 @@ export default function BusinessAttributeRelatedEntity() {
         ]) ||
         [];
 
-    entityData?.isAChildren?.relationships.forEach((businessAttribute) => {
+    entityData?.isAChildren?.relationships?.forEach((businessAttribute) => {
         const childUrn = businessAttribute.entity?.urn;
 
         if (childUrn) {
diff --git a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx
index 913d502972fe1..b20116f615556 100644
--- a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx
+++ b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx
@@ -29,6 +29,7 @@ import { MatchedFieldList } from '../../search/matches/MatchedFieldList';
 import { matchedInputFieldRenderer } from '../../search/matches/matchedInputFieldRenderer';
 import { IncidentTab } from '../shared/tabs/Incident/IncidentTab';
 import { ChartQueryTab } from './ChartQueryTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub Chart entity.
@@ -69,6 +70,8 @@ export class ChartEntity implements Entity<Chart> {
 
     getAutoCompleteFieldName = () => 'title';
 
+    getGraphName = () => 'chart';
+
     getPathName = () => 'chart';
 
     getEntityName = () => 'Chart';
@@ -97,6 +100,9 @@ export class ChartEntity implements Entity<Chart> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderProfile = (urn: string) => (
@@ -136,9 +142,9 @@ export class ChartEntity implements Entity<Chart> {
                     component: EmbedTab,
                     display: {
                         visible: (_, chart: GetChartQuery) =>
-                            !!chart?.chart?.embed?.renderUrl && chart?.chart?.platform.urn === LOOKER_URN,
+                            !!chart?.chart?.embed?.renderUrl && chart?.chart?.platform?.urn === LOOKER_URN,
                         enabled: (_, chart: GetChartQuery) =>
-                            !!chart?.chart?.embed?.renderUrl && chart?.chart?.platform.urn === LOOKER_URN,
+                            !!chart?.chart?.embed?.renderUrl && chart?.chart?.platform?.urn === LOOKER_URN,
                     },
                 },
                 {
@@ -164,7 +170,7 @@ export class ChartEntity implements Entity<Chart> {
                     name: 'Incidents',
                     component: IncidentTab,
                     getDynamicName: (_, chart) => {
-                        const activeIncidentCount = chart?.chart?.activeIncidents.total;
+                        const activeIncidentCount = chart?.chart?.activeIncidents?.total;
                         return `Incidents${(activeIncidentCount && ` (${activeIncidentCount})`) || ''}`;
                     },
                 },
diff --git a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx
index 9cd32cf33a013..557f52146e77a 100644
--- a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx
+++ b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx
@@ -8,7 +8,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'
 import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection';
 import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection';
 import { getDataForEntityType } from '../shared/containers/profile/utils';
-import { useGetContainerQuery } from '../../../graphql/container.generated';
+import { useGetContainerQuery, GetContainerQuery } from '../../../graphql/container.generated';
 import { ContainerEntitiesTab } from './ContainerEntitiesTab';
 import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
@@ -17,6 +17,9 @@ import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { getDataProduct } from '../shared/utils';
 import EmbeddedProfile from '../shared/embed/EmbeddedProfile';
+import AccessManagement from '../shared/tabs/Dataset/AccessManagement/AccessManagement';
+import { useAppConfig } from '../../useAppConfig';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub Container entity.
@@ -57,6 +60,8 @@ export class ContainerEntity implements Entity<Container> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'container';
+
     getPathName = () => 'container';
 
     getEntityName = () => 'Container';
@@ -65,6 +70,8 @@ export class ContainerEntity implements Entity<Container> {
 
     useEntityQuery = useGetContainerQuery;
 
+    appconfig = useAppConfig;
+
     renderProfile = (urn: string) => (
         <EntityProfile
             urn={urn}
@@ -85,6 +92,23 @@ export class ContainerEntity implements Entity<Container> {
                     name: 'Properties',
                     component: PropertiesTab,
                 },
+                {
+                    name: 'Access Management',
+                    component: AccessManagement,
+                    display: {
+                        visible: (_, container: GetContainerQuery) => {
+                            return (
+                                this.appconfig().config.featureFlags.showAccessManagement &&
+                                !!container?.container?.access
+                            );
+                        },
+                        enabled: (_, container: GetContainerQuery) => {
+                            const accessAspect = container?.container?.access;
+                            const rolesList = accessAspect?.roles;
+                            return !!accessAspect && !!rolesList && rolesList.length > 0;
+                        },
+                    },
+                },
             ]}
             sidebarSections={this.getSidebarSections()}
         />
@@ -110,6 +134,9 @@ export class ContainerEntity implements Entity<Container> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
         // TODO: Add back once entity-level recommendations are complete.
         // {
         //    component: SidebarRecommendationsSection,
diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx
index 9564cbc18198e..c35b1f90ee9c0 100644
--- a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx
+++ b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx
@@ -32,6 +32,7 @@ import { LOOKER_URN } from '../../ingest/source/builder/constants';
 import { MatchedFieldList } from '../../search/matches/MatchedFieldList';
 import { matchedInputFieldRenderer } from '../../search/matches/matchedInputFieldRenderer';
 import { IncidentTab } from '../shared/tabs/Incident/IncidentTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub Dashboard entity.
@@ -103,6 +104,9 @@ export class DashboardEntity implements Entity<Dashboard> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderProfile = (urn: string) => (
@@ -145,10 +149,10 @@ export class DashboardEntity implements Entity<Dashboard> {
                     display: {
                         visible: (_, dashboard: GetDashboardQuery) =>
                             !!dashboard?.dashboard?.embed?.renderUrl &&
-                            dashboard?.dashboard?.platform.urn === LOOKER_URN,
+                            dashboard?.dashboard?.platform?.urn === LOOKER_URN,
                         enabled: (_, dashboard: GetDashboardQuery) =>
                             !!dashboard?.dashboard?.embed?.renderUrl &&
-                            dashboard?.dashboard?.platform.urn === LOOKER_URN,
+                            dashboard?.dashboard?.platform?.urn === LOOKER_URN,
                     },
                 },
                 {
@@ -166,7 +170,7 @@ export class DashboardEntity implements Entity<Dashboard> {
                     name: 'Incidents',
                     component: IncidentTab,
                     getDynamicName: (_, dashboard) => {
-                        const activeIncidentCount = dashboard?.dashboard?.activeIncidents.total;
+                        const activeIncidentCount = dashboard?.dashboard?.activeIncidents?.total;
                         return `Incidents${(activeIncidentCount && ` (${activeIncidentCount})`) || ''}`;
                     },
                 },
@@ -291,6 +295,8 @@ export class DashboardEntity implements Entity<Dashboard> {
         ]);
     };
 
+    getGraphName = () => this.getPathName();
+
     renderEmbeddedProfile = (urn: string) => (
         <EmbeddedProfile
             urn={urn}
diff --git a/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx b/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx
index fb5ea28008741..3c03dfb65ccbc 100644
--- a/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx
+++ b/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx
@@ -19,6 +19,7 @@ import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { getDataProduct } from '../shared/utils';
 import { IncidentTab } from '../shared/tabs/Incident/IncidentTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub DataFlow entity.
@@ -53,6 +54,8 @@ export class DataFlowEntity implements Entity<DataFlow> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'dataFlow';
+
     getPathName = () => 'pipelines';
 
     getEntityName = () => 'Pipeline';
@@ -89,7 +92,7 @@ export class DataFlowEntity implements Entity<DataFlow> {
                     name: 'Incidents',
                     component: IncidentTab,
                     getDynamicName: (_, dataFlow) => {
-                        const activeIncidentCount = dataFlow?.dataFlow?.activeIncidents.total;
+                        const activeIncidentCount = dataFlow?.dataFlow?.activeIncidents?.total;
                         return `Incidents${(activeIncidentCount && ` (${activeIncidentCount})`) || ''}`;
                     },
                 },
@@ -121,6 +124,9 @@ export class DataFlowEntity implements Entity<DataFlow> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     getOverridePropertiesFromEntity = (dataFlow?: DataFlow | null): GenericEntityProperties => {
diff --git a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx
index fe1a906371e9d..5b1aaeaef76d5 100644
--- a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx
+++ b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx
@@ -22,6 +22,7 @@ import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { getDataProduct } from '../shared/utils';
 import { IncidentTab } from '../shared/tabs/Incident/IncidentTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 const getDataJobPlatformName = (data?: DataJob): string => {
     return (
@@ -64,6 +65,8 @@ export class DataJobEntity implements Entity<DataJob> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'dataJob';
+
     getPathName = () => 'tasks';
 
     getEntityName = () => 'Task';
@@ -109,7 +112,7 @@ export class DataJobEntity implements Entity<DataJob> {
                     name: 'Incidents',
                     component: IncidentTab,
                     getDynamicName: (_, dataJob) => {
-                        const activeIncidentCount = dataJob?.dataJob?.activeIncidents.total;
+                        const activeIncidentCount = dataJob?.dataJob?.activeIncidents?.total;
                         return `Incidents${(activeIncidentCount && ` (${activeIncidentCount})`) || ''}`;
                     },
                 },
@@ -141,6 +144,9 @@ export class DataJobEntity implements Entity<DataJob> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     getOverridePropertiesFromEntity = (dataJob?: DataJob | null): GenericEntityProperties => {
diff --git a/datahub-web-react/src/app/entity/dataJob/tabs/RunsTab.tsx b/datahub-web-react/src/app/entity/dataJob/tabs/RunsTab.tsx
index 5cd621719ce8f..f00e3402c8ed4 100644
--- a/datahub-web-react/src/app/entity/dataJob/tabs/RunsTab.tsx
+++ b/datahub-web-react/src/app/entity/dataJob/tabs/RunsTab.tsx
@@ -129,8 +129,8 @@ export const RunsTab = () => {
             name: run?.name,
             status: run?.state?.[0]?.status,
             resultType: run?.state?.[0]?.result?.resultType,
-            inputs: run?.inputs?.relationships.map((relationship) => relationship.entity),
-            outputs: run?.outputs?.relationships.map((relationship) => relationship.entity),
+            inputs: run?.inputs?.relationships?.map((relationship) => relationship.entity),
+            outputs: run?.outputs?.relationships?.map((relationship) => relationship.entity),
             externalUrl: run?.externalUrl,
         }));
     if (loading) {
diff --git a/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx b/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx
index 6687ec9f914c1..89cbaf3cbeaa1 100644
--- a/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx
+++ b/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx
@@ -71,4 +71,8 @@ export class DataPlatformEntity implements Entity<DataPlatform> {
     supportedCapabilities = () => {
         return new Set([]);
     };
+
+    getGraphName = () => {
+        return 'dataPlatform';
+    };
 }
diff --git a/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx b/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx
index a542e1b52f510..d0db687ffed92 100644
--- a/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx
+++ b/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx
@@ -58,4 +58,8 @@ export class DataPlatformInstanceEntity implements Entity<DataPlatformInstance>
     supportedCapabilities = () => {
         return new Set([]);
     };
+
+    getGraphName = () => {
+        return 'dataPlatformInstance';
+    };
 }
diff --git a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx
index 6b31de84f85bb..b7912268eb2e3 100644
--- a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx
+++ b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx
@@ -17,6 +17,7 @@ import { DataProductEntitiesTab } from './DataProductEntitiesTab';
 import { EntityActionItem } from '../shared/entity/EntityActions';
 import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub Data Product entity.
@@ -123,6 +124,9 @@ export class DataProductEntity implements Entity<DataProduct> {
                 updateOnly: true,
             },
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderPreview = (_: PreviewType, data: DataProduct) => {
@@ -191,4 +195,8 @@ export class DataProductEntity implements Entity<DataProduct> {
             EntityCapabilityType.DOMAINS,
         ]);
     };
+
+    getGraphName = () => {
+        return 'dataProduct';
+    };
 }
diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
index 21ae085832cb3..abf8e732c47d6 100644
--- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
+++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
@@ -37,6 +37,7 @@ import { matchedFieldPathsRenderer } from '../../search/matches/matchedFieldPath
 import { getLastUpdatedMs } from './shared/utils';
 import { IncidentTab } from '../shared/tabs/Incident/IncidentTab';
 import { GovernanceTab } from '../shared/tabs/Dataset/Governance/GovernanceTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 const SUBTYPES = {
     VIEW: 'view',
@@ -85,6 +86,8 @@ export class DatasetEntity implements Entity<Dataset> {
 
     getPathName = () => 'dataset';
 
+    getGraphName = () => 'dataset';
+
     getEntityName = () => 'Dataset';
 
     getCollectionName = () => 'Datasets';
@@ -214,7 +217,7 @@ export class DatasetEntity implements Entity<Dataset> {
                     name: 'Incidents',
                     component: IncidentTab,
                     getDynamicName: (_, dataset) => {
-                        const activeIncidentCount = dataset?.dataset?.activeIncidents.total;
+                        const activeIncidentCount = dataset?.dataset?.activeIncidents?.total;
                         return `Incidents${(activeIncidentCount && ` (${activeIncidentCount})`) || ''}`;
                     },
                 },
@@ -258,7 +261,11 @@ export class DatasetEntity implements Entity<Dataset> {
         },
         {
             component: DataProductSection,
-        }, // TODO: Add back once entity-level recommendations are complete.
+        },
+        {
+            component: SidebarStructuredPropsSection,
+        },
+        // TODO: Add back once entity-level recommendations are complete.
         // {
         //    component: SidebarRecommendationsSection,
         // },
diff --git a/datahub-web-react/src/app/entity/dataset/profile/OperationsTab.tsx b/datahub-web-react/src/app/entity/dataset/profile/OperationsTab.tsx
index 78ec334f071ba..d3371d3790d02 100644
--- a/datahub-web-react/src/app/entity/dataset/profile/OperationsTab.tsx
+++ b/datahub-web-react/src/app/entity/dataset/profile/OperationsTab.tsx
@@ -195,8 +195,8 @@ export const OperationsTab = () => {
             status: run?.state?.[0]?.status,
             resultType: run?.state?.[0]?.result?.resultType,
             duration: run?.state?.[0]?.durationMillis,
-            inputs: run?.inputs?.relationships.map((relationship) => relationship.entity),
-            outputs: run?.outputs?.relationships.map((relationship) => relationship.entity),
+            inputs: run?.inputs?.relationships?.map((relationship) => relationship.entity),
+            outputs: run?.outputs?.relationships?.map((relationship) => relationship.entity),
             externalUrl: run?.externalUrl,
             parentTemplate: run?.parentTemplate?.relationships?.[0]?.entity,
         }));
diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/StructuredPropValues.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/StructuredPropValues.tsx
new file mode 100644
index 0000000000000..91379ce97fe22
--- /dev/null
+++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/StructuredPropValues.tsx
@@ -0,0 +1,69 @@
+import StructuredPropertyValue from '@src/app/entity/shared/tabs/Properties/StructuredPropertyValue';
+import { mapStructuredPropertyToPropertyRow } from '@src/app/entity/shared/tabs/Properties/useStructuredProperties';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { SchemaFieldEntity, SearchResult, StdDataType } from '@src/types.generated';
+import { Tooltip } from 'antd';
+import React from 'react';
+import styled from 'styled-components';
+
+const ValuesContainer = styled.span`
+    max-width: 120px;
+    display: flex;
+`;
+
+const MoreIndicator = styled.span`
+    float: right;
+`;
+
+interface Props {
+    schemaFieldEntity: SchemaFieldEntity;
+    propColumn: SearchResult | undefined;
+}
+
+const StructuredPropValues = ({ schemaFieldEntity, propColumn }: Props) => {
+    const entityRegistry = useEntityRegistry();
+
+    const property = schemaFieldEntity.structuredProperties?.properties?.find(
+        (prop) => prop.structuredProperty.urn === propColumn?.entity?.urn,
+    );
+    const propRow = property ? mapStructuredPropertyToPropertyRow(property) : undefined;
+    const values = propRow?.values;
+    const isRichText = propRow?.dataType?.info?.type === StdDataType.RichText;
+
+    const hasMoreValues = values && values.length > 2;
+    const displayedValues = hasMoreValues ? values.slice(0, 1) : values;
+    const tooltipContent = values?.map((value) => {
+        const title = value.entity
+            ? entityRegistry.getDisplayName(value.entity.type, value.entity)
+            : value.value?.toString();
+        return <div>{title}</div>;
+    });
+
+    return (
+        <>
+            {values && (
+                <>
+                    {displayedValues?.map((val) => {
+                        return (
+                            <ValuesContainer>
+                                <StructuredPropertyValue
+                                    value={val}
+                                    isRichText={isRichText}
+                                    truncateText
+                                    isFieldColumn
+                                />
+                            </ValuesContainer>
+                        );
+                    })}
+                    {hasMoreValues && (
+                        <Tooltip title={tooltipContent} showArrow={false}>
+                            <MoreIndicator>...</MoreIndicator>
+                        </Tooltip>
+                    )}
+                </>
+            )}
+        </>
+    );
+};
+
+export default StructuredPropValues;
diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx
index 3d03b6306454d..fdd03622fc0e1 100644
--- a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx
+++ b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx
@@ -69,7 +69,7 @@ export default function useSchemaTitleRenderer(
                         ?.filter(
                             (constraint) =>
                                 (constraint?.sourceFields?.filter(
-                                    (sourceField) => sourceField?.fieldPath.trim() === fieldPath.trim(),
+                                    (sourceField) => sourceField?.fieldPath?.trim() === fieldPath.trim(),
                                 ).length || 0) > 0,
                         )
                         .map((constraint) => (
diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts
index 51cb8ab024519..9d719ae25e2be 100644
--- a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts
+++ b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts
@@ -37,7 +37,7 @@ export function convertEditableSchemaMetadataForUpdate(
 ): EditableSchemaMetadataUpdate {
     return {
         editableSchemaFieldInfo:
-            editableSchemaMetadata?.editableSchemaFieldInfo.map((editableSchemaFieldInfo) => ({
+            editableSchemaMetadata?.editableSchemaFieldInfo?.map((editableSchemaFieldInfo) => ({
                 fieldPath: editableSchemaFieldInfo?.fieldPath,
                 description: editableSchemaFieldInfo?.description,
                 globalTags: { tags: convertTagsForUpdate(editableSchemaFieldInfo?.globalTags?.tags || []) },
diff --git a/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx b/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx
index c834597e41a6d..15cc99127f350 100644
--- a/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx
+++ b/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx
@@ -70,7 +70,7 @@ export default function DataProductsTab() {
         },
     });
     const totalResults = data?.searchAcrossEntities?.total || 0;
-    const searchResults = data?.searchAcrossEntities?.searchResults.map((r) => r.entity) || [];
+    const searchResults = data?.searchAcrossEntities?.searchResults?.map((r) => r.entity) || [];
     const dataProducts = [...createdDataProducts, ...searchResults];
     const displayedDataProducts = dataProducts
         .map(
diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx
index 2b67c88a6ff23..0f25f3ce565f0 100644
--- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx
+++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx
@@ -15,6 +15,7 @@ import DataProductsTab from './DataProductsTab/DataProductsTab';
 import { EntityProfileTab } from '../shared/constants';
 import DomainIcon from '../../domain/DomainIcon';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub Domain entity.
@@ -60,6 +61,8 @@ export class DomainEntity implements Entity<Domain> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'domain';
+
     getPathName = () => 'domain';
 
     getEntityName = () => 'Domain';
@@ -110,6 +113,9 @@ export class DomainEntity implements Entity<Domain> {
         {
             component: SidebarOwnerSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderPreview = (_: PreviewType, data: Domain) => {
diff --git a/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx b/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx
index aece3db1312af..3eb950cb0e7ac 100644
--- a/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx
+++ b/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx
@@ -58,6 +58,8 @@ export class ERModelRelationshipEntity implements Entity<ErModelRelationship> {
 
     getEntityName = () => 'ER-Model-Relationship';
 
+    getGraphName = () => 'erModelRelationship';
+
     renderProfile = (urn: string) => (
         <EntityProfile
             urn={urn}
diff --git a/datahub-web-react/src/app/entity/glossaryNode/GlossaryNodeEntity.tsx b/datahub-web-react/src/app/entity/glossaryNode/GlossaryNodeEntity.tsx
index 803c703b7f4c5..861e96811b340 100644
--- a/datahub-web-react/src/app/entity/glossaryNode/GlossaryNodeEntity.tsx
+++ b/datahub-web-react/src/app/entity/glossaryNode/GlossaryNodeEntity.tsx
@@ -12,6 +12,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'
 import ChildrenTab from './ChildrenTab';
 import { Preview } from './preview/Preview';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 class GlossaryNodeEntity implements Entity<GlossaryNode> {
     type: EntityType = EntityType.GlossaryNode;
@@ -100,6 +101,9 @@ class GlossaryNodeEntity implements Entity<GlossaryNode> {
         {
             component: SidebarOwnerSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     displayName = (data: GlossaryNode) => {
@@ -147,6 +151,8 @@ class GlossaryNodeEntity implements Entity<GlossaryNode> {
             EntityCapabilityType.SOFT_DELETE,
         ]);
     };
+
+    getGraphName = () => this.getPathName();
 }
 
 export default GlossaryNodeEntity;
diff --git a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx
index 8bbc0a693b223..439cba2ea6923 100644
--- a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx
+++ b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx
@@ -18,6 +18,7 @@ import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import { EntityActionItem } from '../shared/entity/EntityActions';
 import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection';
 import { PageRoutes } from '../../../conf/Global';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub Dataset entity.
@@ -129,6 +130,9 @@ export class GlossaryTermEntity implements Entity<GlossaryTerm> {
                 hideOwnerType: true,
             },
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     getOverridePropertiesFromEntity = (glossaryTerm?: GlossaryTerm | null): GenericEntityProperties => {
@@ -179,4 +183,6 @@ export class GlossaryTermEntity implements Entity<GlossaryTerm> {
             EntityCapabilityType.SOFT_DELETE,
         ]);
     };
+
+    getGraphName = () => this.getPathName();
 }
diff --git a/datahub-web-react/src/app/entity/glossaryTerm/profile/AddRelatedTermsModal.tsx b/datahub-web-react/src/app/entity/glossaryTerm/profile/AddRelatedTermsModal.tsx
index a609dc4cca599..9131af265a751 100644
--- a/datahub-web-react/src/app/entity/glossaryTerm/profile/AddRelatedTermsModal.tsx
+++ b/datahub-web-react/src/app/entity/glossaryTerm/profile/AddRelatedTermsModal.tsx
@@ -104,7 +104,10 @@ function AddRelatedTermsModal(props: Props) {
         const newUrns = [...selectedUrns, urn];
         setSelectedUrns(newUrns);
         const selectedSearchOption = tagSearchOptions.find((option) => option.props.value === urn);
-        setSelectedTerms([...selectedTerms, { urn, component: <TermLabel name={selectedSearchOption?.props.name} /> }]);
+        setSelectedTerms([
+            ...selectedTerms,
+            { urn, component: <TermLabel name={selectedSearchOption?.props?.name} /> },
+        ]);
     };
 
     // When a Tag or term search result is deselected, remove the urn from the Owners
diff --git a/datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx b/datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx
index 098e97e526fd8..6959fc082f734 100644
--- a/datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx
+++ b/datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx
@@ -22,7 +22,7 @@ export default function GlossaryRelatedEntity() {
         ]) ||
         [];
 
-    entityData?.isAChildren?.relationships.forEach((term) => {
+    entityData?.isAChildren?.relationships?.forEach((term) => {
         const childUrn = term.entity?.urn;
 
         if (childUrn) {
diff --git a/datahub-web-react/src/app/entity/group/Group.tsx b/datahub-web-react/src/app/entity/group/Group.tsx
index cd9cf1ca6eec4..763db856f33ac 100644
--- a/datahub-web-react/src/app/entity/group/Group.tsx
+++ b/datahub-web-react/src/app/entity/group/Group.tsx
@@ -40,6 +40,8 @@ export class GroupEntity implements Entity<CorpGroup> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName: () => string = () => 'corpGroup';
+
     getPathName: () => string = () => 'group';
 
     getEntityName = () => 'Group';
diff --git a/datahub-web-react/src/app/entity/group/GroupMembers.tsx b/datahub-web-react/src/app/entity/group/GroupMembers.tsx
index 147c3f8030d0e..28e81b438d4cb 100644
--- a/datahub-web-react/src/app/entity/group/GroupMembers.tsx
+++ b/datahub-web-react/src/app/entity/group/GroupMembers.tsx
@@ -137,12 +137,13 @@ export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeM
         }, 3000);
     };
 
-    const onRemoveMember = (memberUrn: string) => {
+    const onRemoveMember = (memberEntity: CorpUser) => {
+        const memberName = entityRegistry.getDisplayName(EntityType.CorpUser, memberEntity);
         Modal.confirm({
             title: `Confirm Group Member Removal`,
-            content: `Are you sure you want to remove this user from the group?`,
+            content: `Are you sure you want to remove ${memberName} user from the group?`,
             onOk() {
-                removeGroupMember(memberUrn);
+                removeGroupMember(memberEntity?.urn);
             },
             onCancel() {},
             okText: 'Yes',
@@ -155,7 +156,7 @@ export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeM
     const total = relationships?.total || 0;
     const groupMembers = relationships?.relationships?.map((rel) => rel.entity as CorpUser) || [];
 
-    const getItems = (urnID: string): MenuProps['items'] => {
+    const getItems = (userEntity: CorpUser): MenuProps['items'] => {
         return [
             {
                 key: 'make',
@@ -169,7 +170,7 @@ export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeM
             {
                 key: 'remove',
                 disabled: isExternalGroup,
-                onClick: () => onRemoveMember(urnID),
+                onClick: () => onRemoveMember(userEntity),
                 label: (
                     <span>
                         <UserDeleteOutlined /> Remove from Group
@@ -210,7 +211,7 @@ export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeM
                                 </MemberColumn>
                                 <MemberColumn xl={1} lg={1} md={1} sm={1} xs={1}>
                                     <MemberEditIcon>
-                                        <Dropdown menu={{ items: getItems(item.urn) }}>
+                                        <Dropdown menu={{ items: getItems(item) }}>
                                             <MoreOutlined />
                                         </Dropdown>
                                     </MemberEditIcon>
diff --git a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx
index 2f2786b1c0d96..51b66c8c2a41d 100644
--- a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx
+++ b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx
@@ -18,6 +18,7 @@ import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { getDataProduct } from '../shared/utils';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub MLFeature entity.
@@ -52,6 +53,8 @@ export class MLFeatureEntity implements Entity<MlFeature> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'mlFeature';
+
     getPathName = () => 'features';
 
     getEntityName = () => 'Feature';
@@ -120,6 +123,9 @@ export class MLFeatureEntity implements Entity<MlFeature> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderPreview = (_: PreviewType, data: MlFeature) => {
diff --git a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx
index 595c73fbc3cb6..56d4622311fb1 100644
--- a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx
+++ b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx
@@ -19,6 +19,7 @@ import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { getDataProduct } from '../shared/utils';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub MLFeatureTable entity.
@@ -53,6 +54,8 @@ export class MLFeatureTableEntity implements Entity<MlFeatureTable> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'mlFeatureTable';
+
     getPathName = () => 'featureTables';
 
     getEntityName = () => 'Feature Table';
@@ -88,6 +91,9 @@ export class MLFeatureTableEntity implements Entity<MlFeatureTable> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderProfile = (urn: string) => (
diff --git a/datahub-web-react/src/app/entity/mlFeatureTable/profile/Sources.tsx b/datahub-web-react/src/app/entity/mlFeatureTable/profile/Sources.tsx
index eb114e9774e29..56c0fcbb60200 100644
--- a/datahub-web-react/src/app/entity/mlFeatureTable/profile/Sources.tsx
+++ b/datahub-web-react/src/app/entity/mlFeatureTable/profile/Sources.tsx
@@ -35,14 +35,14 @@ export default function SourcesView() {
             features?.reduce((accumulator: Array<Dataset>, feature) => {
                 if (feature.__typename === 'MLFeature' && feature.properties?.sources) {
                     // eslint-disable-next-line array-callback-return
-                    feature.properties?.sources.map((source: Dataset | null) => {
+                    feature.properties?.sources?.map((source: Dataset | null) => {
                         if (source && accumulator.findIndex((dataset) => dataset.urn === source?.urn) === -1) {
                             accumulator.push(source);
                         }
                     });
                 } else if (feature.__typename === 'MLPrimaryKey' && feature.properties?.sources) {
                     // eslint-disable-next-line array-callback-return
-                    feature.properties?.sources.map((source: Dataset | null) => {
+                    feature.properties?.sources?.map((source: Dataset | null) => {
                         if (source && accumulator.findIndex((dataset) => dataset.urn === source?.urn) === -1) {
                             accumulator.push(source);
                         }
diff --git a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx
index d4d0b37da9ec9..b77f6a19436a5 100644
--- a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx
+++ b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx
@@ -18,6 +18,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'
 import MlModelFeaturesTab from './profile/MlModelFeaturesTab';
 import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub MlModel entity.
@@ -52,6 +53,8 @@ export class MLModelEntity implements Entity<MlModel> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'mlModel';
+
     getPathName = () => 'mlModels';
 
     getEntityName = () => 'ML Model';
@@ -89,6 +92,9 @@ export class MLModelEntity implements Entity<MlModel> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderProfile = (urn: string) => (
diff --git a/datahub-web-react/src/app/entity/mlModel/profile/MlModelFeaturesTab.tsx b/datahub-web-react/src/app/entity/mlModel/profile/MlModelFeaturesTab.tsx
index b8dc64793c225..7fd719177d151 100644
--- a/datahub-web-react/src/app/entity/mlModel/profile/MlModelFeaturesTab.tsx
+++ b/datahub-web-react/src/app/entity/mlModel/profile/MlModelFeaturesTab.tsx
@@ -9,7 +9,7 @@ export default function MlModelFeaturesTab() {
     const entity = useBaseEntity() as GetMlModelQuery;
 
     const model = entity && entity.mlModel;
-    const features = model?.features?.relationships.map((relationship) => relationship.entity) as Array<
+    const features = model?.features?.relationships?.map((relationship) => relationship.entity) as Array<
         MlFeature | MlPrimaryKey
     >;
 
diff --git a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx
index 5896c1864cc43..5c820007fd1e2 100644
--- a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx
+++ b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx
@@ -16,6 +16,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'
 import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub MlModelGroup entity.
@@ -50,6 +51,8 @@ export class MLModelGroupEntity implements Entity<MlModelGroup> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'mlModelGroup';
+
     getPathName = () => 'mlModelGroup';
 
     getEntityName = () => 'ML Group';
@@ -85,6 +88,9 @@ export class MLModelGroupEntity implements Entity<MlModelGroup> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderProfile = (urn: string) => (
diff --git a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx
index 60c7531a4f57c..d72fabc17ecf6 100644
--- a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx
+++ b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx
@@ -17,6 +17,7 @@ import { LineageTab } from '../shared/tabs/Lineage/LineageTab';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { getDataProduct } from '../shared/utils';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
+import SidebarStructuredPropsSection from '../shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 
 /**
  * Definition of the DataHub MLPrimaryKey entity.
@@ -51,6 +52,8 @@ export class MLPrimaryKeyEntity implements Entity<MlPrimaryKey> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'mlPrimaryKey';
+
     getPathName = () => 'mlPrimaryKeys';
 
     getEntityName = () => 'ML Primary Key';
@@ -118,6 +121,9 @@ export class MLPrimaryKeyEntity implements Entity<MlPrimaryKey> {
         {
             component: DataProductSection,
         },
+        {
+            component: SidebarStructuredPropsSection,
+        },
     ];
 
     renderPreview = (_: PreviewType, data: MlPrimaryKey) => {
diff --git a/datahub-web-react/src/app/entity/ownership/OwnershipList.tsx b/datahub-web-react/src/app/entity/ownership/OwnershipList.tsx
index 8d6e109d2bca3..3d7904a5c5505 100644
--- a/datahub-web-react/src/app/entity/ownership/OwnershipList.tsx
+++ b/datahub-web-react/src/app/entity/ownership/OwnershipList.tsx
@@ -64,7 +64,7 @@ export const OwnershipList = () => {
     });
     const totalOwnershipTypes = data?.listOwnershipTypes?.total || 0;
     const ownershipTypes =
-        data?.listOwnershipTypes?.ownershipTypes.filter((type) => type.urn !== 'urn:li:ownershipType:none') || [];
+        data?.listOwnershipTypes?.ownershipTypes?.filter((type) => type.urn !== 'urn:li:ownershipType:none') || [];
 
     const onClickCreateOwnershipType = () => {
         setShowOwnershipBuilder(true);
diff --git a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx
index 7e74b43e68afb..fdc0e33d77a05 100644
--- a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx
+++ b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx
@@ -1,9 +1,10 @@
 import * as React from 'react';
 import { PicCenterOutlined } from '@ant-design/icons';
-import { EntityType, SchemaFieldEntity, SearchResult } from '../../../types.generated';
+import { Dataset, EntityType, SchemaFieldEntity, SearchResult } from '../../../types.generated';
 import { Entity, IconStyleType, PreviewType } from '../Entity';
 import { getDataForEntityType } from '../shared/containers/profile/utils';
 import { Preview } from './preview/Preview';
+import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 
 export class SchemaFieldPropertiesEntity implements Entity<SchemaFieldEntity> {
     type: EntityType = EntityType.SchemaField;
@@ -18,24 +19,48 @@ export class SchemaFieldPropertiesEntity implements Entity<SchemaFieldEntity> {
 
     isLineageEnabled = () => false;
 
+    getParentDataset = (parent) => {
+        return {
+            urn: parent?.urn,
+            name: parent?.name,
+            type: parent?.type,
+            platform: parent?.platfrom,
+            properties: parent?.properties,
+        } as Dataset;
+    };
+
     // Currently unused.
     getAutoCompleteFieldName = () => 'schemaField';
 
     // Currently unused.
     getPathName = () => 'schemaField';
 
-    // Currently unused.
-    getEntityName = () => 'schemaField';
+    getEntityName = () => 'Column';
 
-    // Currently unused.
-    getCollectionName = () => 'schemaFields';
+    getCollectionName = () => 'Columns';
 
     // Currently unused.
     renderProfile = (_: string) => <></>;
 
-    renderPreview = (previewType: PreviewType, data: SchemaFieldEntity) => (
-        <Preview previewType={previewType} datasetUrn={data.parent.urn} name={data.fieldPath} />
-    );
+    getGraphName = () => 'schemaField';
+
+    renderPreview = (previewType: PreviewType, data: SchemaFieldEntity) => {
+        const parent = data.parent as Dataset;
+        return (
+            <Preview
+                previewType={previewType}
+                datasetUrn={data.parent.urn}
+                name={data.fieldPath}
+                parentContainers={parent?.parentContainers}
+                platformName={
+                    parent?.platform?.properties?.displayName || capitalizeFirstLetterOnly(parent?.platform?.name)
+                }
+                platformLogo={parent?.platform?.properties?.logoUrl || ''}
+                platformInstanceId={parent?.dataPlatformInstance?.instanceId}
+                parentDataset={this.getParentDataset(parent)}
+            />
+        );
+    };
 
     renderSearch = (result: SearchResult) => this.renderPreview(PreviewType.SEARCH, result.entity as SchemaFieldEntity);
 
diff --git a/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx b/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx
index b22e988c76672..086b1a3802188 100644
--- a/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx
+++ b/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx
@@ -1,6 +1,6 @@
 import React from 'react';
 import { PicCenterOutlined } from '@ant-design/icons';
-import { EntityType, Owner } from '../../../../types.generated';
+import { Dataset, EntityType, Owner, ParentContainersResult } from '../../../../types.generated';
 import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
 import { useEntityRegistry } from '../../../useEntityRegistry';
 import { IconStyleType, PreviewType } from '../../Entity';
@@ -11,12 +11,22 @@ export const Preview = ({
     description,
     owners,
     previewType,
+    parentContainers,
+    platformName,
+    platformLogo,
+    platformInstanceId,
+    parentDataset,
 }: {
     datasetUrn: string;
     name: string;
     description?: string | null;
     owners?: Array<Owner> | null;
     previewType: PreviewType;
+    parentContainers?: ParentContainersResult | null;
+    platformName?: string;
+    platformLogo?: string | null;
+    platformInstanceId?: string;
+    parentDataset?: Dataset;
 }): JSX.Element => {
     const entityRegistry = useEntityRegistry();
 
@@ -35,6 +45,11 @@ export const Preview = ({
             logoComponent={<PicCenterOutlined style={{ fontSize: '20px' }} />}
             type="Column"
             typeIcon={entityRegistry.getIcon(EntityType.SchemaField, 14, IconStyleType.ACCENT)}
+            logoUrl={platformLogo || ''}
+            platform={platformName}
+            platformInstanceId={platformInstanceId}
+            parentContainers={parentContainers}
+            parentDataset={parentDataset}
         />
     );
 };
diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx
index d43b04ec11a16..1e918757e4619 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx
@@ -23,7 +23,7 @@ export function filterResultsForMove(entity: Domain, entityUrn: string) {
     return (
         entity.urn !== entityUrn &&
         entity.__typename === 'Domain' &&
-        !entity.parentDomains?.domains.some((node) => node.urn === entityUrn)
+        !entity.parentDomains?.domains?.some((node) => node.urn === entityUrn)
     );
 }
 
diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx
index 7227354a46569..3746c5f6b95f8 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx
@@ -22,7 +22,7 @@ export function filterResultsForMove(entity: GlossaryNode, entityUrn: string) {
     return (
         entity.urn !== entityUrn &&
         entity.__typename === 'GlossaryNode' &&
-        !entity.parentNodes?.nodes.some((node) => node.urn === entityUrn)
+        !entity.parentNodes?.nodes?.some((node) => node.urn === entityUrn)
     );
 }
 
diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts
index 81f19331e18b7..2d9c346e18d5b 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts
@@ -13,7 +13,7 @@ export function useHandleMoveDomainComplete() {
         if (!entityData) return;
 
         const domain = entityData as Domain;
-        const oldParentUrn = domain.parentDomains?.domains.length ? domain.parentDomains.domains[0].urn : undefined;
+        const oldParentUrn = domain.parentDomains?.domains?.length ? domain.parentDomains.domains[0].urn : undefined;
 
         analytics.event({
             type: EventType.MoveDomainEvent,
diff --git a/datahub-web-react/src/app/entity/shared/PreviewContext.tsx b/datahub-web-react/src/app/entity/shared/PreviewContext.tsx
new file mode 100644
index 0000000000000..889a6726f3c04
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/PreviewContext.tsx
@@ -0,0 +1,9 @@
+import React from 'react';
+import { GenericEntityProperties } from './types';
+
+const PreviewContext = React.createContext<GenericEntityProperties | null>(null);
+export default PreviewContext;
+
+export function usePreviewData() {
+    return React.useContext(PreviewContext);
+}
diff --git a/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts b/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts
index 00e89e5943c17..f3b857ab2ef3b 100644
--- a/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts
+++ b/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts
@@ -203,10 +203,10 @@ describe('siblingUtils', () => {
 
             // merges schema metadata properly  by fieldPath
             expect(combinedData.dataset.schemaMetadata?.fields).toHaveLength(4);
-            expect(combinedData.dataset.schemaMetadata?.fields[0].fieldPath).toEqual('new_one');
-            expect(combinedData.dataset.schemaMetadata?.fields[1].fieldPath).toEqual('DUPLICATE_FIELD');
-            expect(combinedData.dataset.schemaMetadata?.fields[2].fieldPath).toEqual('user_id');
-            expect(combinedData.dataset.schemaMetadata?.fields[3].fieldPath).toEqual('user_name');
+            expect(combinedData.dataset.schemaMetadata?.fields[0]?.fieldPath).toEqual('new_one');
+            expect(combinedData.dataset.schemaMetadata?.fields[1]?.fieldPath).toEqual('DUPLICATE_FIELD');
+            expect(combinedData.dataset.schemaMetadata?.fields[2]?.fieldPath).toEqual('user_id');
+            expect(combinedData.dataset.schemaMetadata?.fields[3]?.fieldPath).toEqual('user_name');
 
             // will overwrite string properties w/ primary
             expect(combinedData.dataset.editableProperties.description).toEqual('secondary description');
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx
index 08e9636f760de..613264709ac23 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx
@@ -19,8 +19,6 @@ const DeprecatedContainer = styled.div`
     justify-content: center;
     align-items: center;
     color: #cd0d24;
-    margin-left: 0px;
-    margin-right: 8px;
     padding-top: 8px;
     padding-bottom: 8px;
     padding-right: 4px;
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.tsx
index ea40fb7181803..de22daeeedd81 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.tsx
@@ -241,9 +241,9 @@ export const CreateERModelRelationModal = ({
     };
 
     const table1NameBusiness = getDatasetName(table1Dataset);
-    const table1NameTech = table1Dataset?.name || table1Dataset?.urn.split(',').at(1) || '';
+    const table1NameTech = table1Dataset?.name || table1Dataset?.urn?.split(',').at(1) || '';
     const table2NameBusiness = getDatasetName(table2Dataset);
-    const table2NameTech = table2Dataset?.name || table2Dataset?.urn.split(',').at(1) || '';
+    const table2NameTech = table2Dataset?.name || table2Dataset?.urn?.split(',').at(1) || '';
 
     const handleAdd = () => {
         const newData: ERModelRelationDataType = {
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationUtils.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationUtils.tsx
index e7a242daa5121..811ebf99b123a 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationUtils.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationUtils.tsx
@@ -68,6 +68,6 @@ export function getDatasetName(datainput: any): string {
         datainput?.editableProperties?.name ||
         datainput?.properties?.name ||
         datainput?.name ||
-        datainput?.urn.split(',').at(1)
+        datainput?.urn?.split(',')?.at(1)
     );
 }
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/EditableCell.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/EditableCell.tsx
index a190d9090cc8c..4320aacd53e4a 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/EditableCell.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/EditableCell.tsx
@@ -53,7 +53,7 @@ export const EditableCell = ({
                 <Select
                     size="large"
                     className="ermodelrelation-select-selector"
-                    options={tableRecord?.schemaMetadata?.fields.map((result) => ({
+                    options={tableRecord?.schemaMetadata?.fields?.map((result) => ({
                         value: result.fieldPath,
                         label: result.fieldPath,
                     }))}
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleStringInput.tsx b/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleOpenEndedInput.tsx
similarity index 87%
rename from datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleStringInput.tsx
rename to datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleOpenEndedInput.tsx
index fe6c0bbb99ce2..fe6cd1115419a 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleStringInput.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleOpenEndedInput.tsx
@@ -4,6 +4,8 @@ import React from 'react';
 import styled from 'styled-components';
 import { ANTD_GRAY_V2 } from '../../../constants';
 
+const MultiStringWrapper = styled.div``;
+
 const StyledInput = styled(Input)`
     width: 75%;
     min-width: 350px;
@@ -29,10 +31,11 @@ const DeleteButton = styled(Button)`
 
 interface Props {
     selectedValues: any[];
+    inputType?: string;
     updateSelectedValues: (values: any[]) => void;
 }
 
-export default function MultipleStringInput({ selectedValues, updateSelectedValues }: Props) {
+export default function MultipleOpenEndedInput({ selectedValues, updateSelectedValues, inputType = 'text' }: Props) {
     function updateInput(text: string, index: number) {
         const updatedValues =
             selectedValues.length > 0 ? selectedValues.map((value, i) => (i === index ? text : value)) : [text];
@@ -53,14 +56,14 @@ export default function MultipleStringInput({ selectedValues, updateSelectedValu
     }
 
     return (
-        <div>
+        <MultiStringWrapper>
             {selectedValues.length > 1 &&
                 selectedValues.map((selectedValue, index) => {
                     const key = `${index}`;
                     return (
                         <InputWrapper key={key}>
                             <StyledInput
-                                type="text"
+                                type={inputType}
                                 value={selectedValue}
                                 onChange={(e) => updateInput(e.target.value, index)}
                             />
@@ -70,7 +73,7 @@ export default function MultipleStringInput({ selectedValues, updateSelectedValu
                 })}
             {selectedValues.length <= 1 && (
                 <StyledInput
-                    type="text"
+                    type={inputType}
                     value={selectedValues[0] || ''}
                     onChange={(e) => updateInput(e.target.value, 0)}
                 />
@@ -78,6 +81,6 @@ export default function MultipleStringInput({ selectedValues, updateSelectedValu
             <StyledButton type="link" onClick={addNewValue}>
                 + Add More
             </StyledButton>
-        </div>
+        </MultiStringWrapper>
     );
 }
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/NumberInput.tsx b/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/NumberInput.tsx
index c56d85db7ef71..f4cedc4cf80ee 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/NumberInput.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/NumberInput.tsx
@@ -1,7 +1,9 @@
 import { Input } from 'antd';
 import React, { ChangeEvent } from 'react';
 import styled from 'styled-components';
+import { PropertyCardinality } from '@src/types.generated';
 import { ANTD_GRAY_V2 } from '../../../constants';
+import MultipleOpenEndedInput from './MultipleOpenEndedInput';
 
 const StyledInput = styled(Input)`
     border: 1px solid ${ANTD_GRAY_V2[6]};
@@ -10,15 +12,31 @@ const StyledInput = styled(Input)`
 
 interface Props {
     selectedValues: any[];
+    cardinality?: PropertyCardinality | null;
     updateSelectedValues: (values: string[] | number[]) => void;
 }
 
-export default function NumberInput({ selectedValues, updateSelectedValues }: Props) {
+export default function NumberInput({ selectedValues, cardinality, updateSelectedValues }: Props) {
     function updateInput(event: ChangeEvent<HTMLInputElement>) {
         const number = Number(event.target.value);
         updateSelectedValues([number]);
     }
 
+    function updateMultipleValues(values: string[] | number[]) {
+        const numbers = values.map((v) => Number(v));
+        updateSelectedValues(numbers);
+    }
+
+    if (cardinality === PropertyCardinality.Multiple) {
+        return (
+            <MultipleOpenEndedInput
+                selectedValues={selectedValues}
+                updateSelectedValues={updateMultipleValues}
+                inputType="number"
+            />
+        );
+    }
+
     return (
         <StyledInput
             type="number"
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StringInput.tsx b/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StringInput.tsx
index 8c69174a35bf3..47912dbc22dd1 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StringInput.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StringInput.tsx
@@ -3,7 +3,7 @@ import React, { ChangeEvent } from 'react';
 import styled from 'styled-components';
 import { ANTD_GRAY_V2 } from '../../../constants';
 import { PropertyCardinality } from '../../../../../../types.generated';
-import MultipleStringInput from './MultipleStringInput';
+import MultipleOpenEndedInput from './MultipleOpenEndedInput';
 
 const StyledInput = styled(Input)`
     width: 75%;
@@ -24,8 +24,15 @@ export default function StringInput({ selectedValues, cardinality, updateSelecte
     }
 
     if (cardinality === PropertyCardinality.Multiple) {
-        return <MultipleStringInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />;
+        return <MultipleOpenEndedInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />;
     }
 
-    return <StyledInput type="text" value={selectedValues[0] || ''} onChange={updateInput} />;
+    return (
+        <StyledInput
+            type="text"
+            value={selectedValues[0] || ''}
+            onChange={updateInput}
+            data-testid="structured-property-string-value-input"
+        />
+    );
 }
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StructuredPropertyInput.tsx b/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StructuredPropertyInput.tsx
index 894a304335b0f..305347ee0bce8 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StructuredPropertyInput.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StructuredPropertyInput.tsx
@@ -60,7 +60,11 @@ export default function StructuredPropertyInput({
                 <DateInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />
             )}
             {!allowedValues && valueType.info.type === StdDataType.Number && (
-                <NumberInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />
+                <NumberInput
+                    selectedValues={selectedValues}
+                    cardinality={cardinality}
+                    updateSelectedValues={updateSelectedValues}
+                />
             )}
             {!allowedValues && valueType.info.type === StdDataType.Urn && (
                 <UrnInput
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
index a9737c9698f7b..1deeed076d8d6 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
@@ -323,7 +323,13 @@ export const EntityProfile = <T, U>({
                 {showBrowseBar && <EntityProfileNavBar urn={urn} entityType={entityType} />}
                 {entityData?.status?.removed === true && (
                     <Alert
-                        message="This entity is not discoverable via search or lineage graph. Contact your DataHub admin for more information."
+                        message={
+                            <>
+                                This entity is marked as soft-deleted, likely due to stateful ingestion or a manual
+                                deletion command, and will not appear in search or lineage graphs. Contact your DataHub
+                                admin for more information.
+                            </>
+                        }
                         banner
                     />
                 )}
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx
index 12fa9131f33c7..7dd2888ba14f7 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx
@@ -18,11 +18,13 @@ import { useUserContext } from '../../../../../context/useUserContext';
 import { useEntityRegistry } from '../../../../../useEntityRegistry';
 import EntityHeaderLoadingSection from './EntityHeaderLoadingSection';
 import { useIsEditableDatasetNameEnabled } from '../../../../../useAppConfig';
+import StructuredPropertyBadge from './StructuredPropertyBadge';
 
 const TitleWrapper = styled.div`
     display: flex;
     justify-content: left;
     align-items: center;
+    gap: 8px;
 
     .ant-typography-edit-content {
         padding-top: 7px;
@@ -132,6 +134,7 @@ export const EntityHeader = ({ headerDropdownItems, headerActionItems, isNameEdi
                                         baseUrl={entityRegistry.getEntityUrl(entityType, urn)}
                                     />
                                 )}
+                                <StructuredPropertyBadge structuredProperties={entityData?.structuredProperties} />
                             </TitleWrapper>
                             <EntityCount
                                 entityCount={entityCount}
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx
index 702f780f1aa11..8976629d9ef0b 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx
@@ -8,8 +8,6 @@ import { useEntityData, useRefetch } from '../../../EntityContext';
 import { useGlossaryEntityData } from '../../../GlossaryEntityContext';
 
 export const EntityTitle = styled(Typography.Title)`
-    margin-right: 10px;
-
     &&& {
         margin-bottom: 0;
         word-break: break-all;
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/DatasetLink.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/DatasetLink.tsx
new file mode 100644
index 0000000000000..cf4891acd384d
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/DatasetLink.tsx
@@ -0,0 +1,46 @@
+import React from 'react';
+import styled from 'styled-components';
+import { Link } from 'react-router-dom';
+import { Typography } from 'antd';
+import { Dataset, EntityType } from '../../../../../../../types.generated';
+import { ANTD_GRAY } from '../../../../constants';
+import { useEntityRegistry } from '../../../../../../useEntityRegistry';
+import { IconStyleType } from '../../../../../Entity';
+
+const DatasetText = styled(Typography.Text)`
+    font-size: 12px;
+    line-height: 20px;
+    color: ${ANTD_GRAY[7]};
+`;
+
+export const DatasetIcon = styled.span`
+    color: ${ANTD_GRAY[7]};
+    margin-right: 4px;
+    font-size: 12px;
+`;
+
+const StyledLink = styled(Link)`
+    white-space: nowrap;
+`;
+
+interface Props {
+    parentDataset: Dataset;
+}
+
+function DatasetLink(props: Props) {
+    const { parentDataset } = props;
+    const entityRegistry = useEntityRegistry();
+    if (!parentDataset) return null;
+
+    const datasetUrl = entityRegistry.getEntityUrl(EntityType.Dataset, parentDataset.urn);
+    const datasetName = entityRegistry.getDisplayName(EntityType.Dataset, parentDataset);
+
+    return (
+        <StyledLink to={datasetUrl} data-testid="dataset">
+            <DatasetIcon>{entityRegistry.getIcon(EntityType.Dataset, 14, IconStyleType.ACCENT)}</DatasetIcon>
+            <DatasetText>{datasetName}</DatasetText>
+        </StyledLink>
+    );
+}
+
+export default DatasetLink;
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx
index 3104dfd911aab..70cc5fa3b36d1 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx
@@ -2,9 +2,10 @@ import React from 'react';
 import styled from 'styled-components';
 import { Typography, Image } from 'antd';
 import { Maybe } from 'graphql/jsutils/Maybe';
-import { Container, Entity } from '../../../../../../../types.generated';
+import { Container, Dataset, Entity } from '../../../../../../../types.generated';
 import { ANTD_GRAY } from '../../../../constants';
 import ContainerLink from './ContainerLink';
+import DatasetLink from './DatasetLink';
 import {
     StyledRightOutlined,
     ParentNodesWrapper as ParentContainersWrapper,
@@ -80,6 +81,7 @@ interface Props {
     parentEntities?: Entity[] | null;
     parentContainersRef: React.RefObject<HTMLDivElement>;
     areContainersTruncated: boolean;
+    parentDataset?: Dataset;
 }
 
 function PlatformContentView(props: Props) {
@@ -96,6 +98,7 @@ function PlatformContentView(props: Props) {
         parentContainers,
         parentContainersRef,
         areContainersTruncated,
+        parentDataset,
     } = props;
 
     const directParentContainer = parentContainers && parentContainers[0];
@@ -152,6 +155,12 @@ function PlatformContentView(props: Props) {
                 </ParentContainersWrapper>
                 {directParentContainer && <ContainerLink container={directParentContainer} />}
             </StyledTooltip>
+            {parentDataset && (
+                <span>
+                    <StyledRightOutlined data-testid="right-arrow" />
+                    <DatasetLink parentDataset={parentDataset} />
+                </span>
+            )}
             <ParentEntities parentEntities={parentEntities || []} numVisible={3} />
         </PlatformContentWrapper>
     );
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/StructuredPropertyBadge.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/StructuredPropertyBadge.tsx
new file mode 100644
index 0000000000000..fc7892f6ba6cc
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/StructuredPropertyBadge.tsx
@@ -0,0 +1,92 @@
+import { colors, Pill, Text, Tooltip } from '@src/alchemy-components';
+import { getStructuredPropertyValue } from '@src/app/entity/shared/utils';
+import { getDisplayName } from '@src/app/govern/structuredProperties/utils';
+import { StructuredProperties } from '@src/types.generated';
+import React from 'react';
+import styled from 'styled-components';
+import { mapStructuredPropertyToPropertyRow } from '../../../tabs/Properties/useStructuredProperties';
+import { filterForAssetBadge } from './utils';
+
+export const MAX_PROP_BADGE_WIDTH = 150;
+
+const StyledTooltip = styled(Tooltip)`
+    .ant-tooltip-inner {
+        border-radius: 8px;
+        box-shadow: 0px 4px 12px 0px rgba(9, 1, 61, 0.12);
+    }
+`;
+
+const TooltipContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    gap: 16px;
+`;
+
+const ValueContainer = styled.div`
+    gap: 4px;
+`;
+
+const BadgeContainer = styled.div`
+    max-width: ${MAX_PROP_BADGE_WIDTH}px;
+`;
+
+interface Props {
+    structuredProperties?: StructuredProperties | null;
+}
+
+const StructuredPropertyBadge = ({ structuredProperties }: Props) => {
+    const badgeStructuredProperty = structuredProperties?.properties?.find(filterForAssetBadge);
+
+    const propRow = badgeStructuredProperty ? mapStructuredPropertyToPropertyRow(badgeStructuredProperty) : undefined;
+
+    if (!badgeStructuredProperty) return null;
+
+    const propertyValue = propRow?.values[0]?.value;
+    const relatedDescription = propRow?.structuredProperty?.definition?.allowedValues?.find(
+        (v) => getStructuredPropertyValue(v.value) === propertyValue,
+    )?.description;
+
+    const BadgeTooltip = () => {
+        return (
+            <TooltipContainer>
+                <Text color="gray" weight="semiBold">
+                    {getDisplayName(badgeStructuredProperty.structuredProperty)}
+                </Text>
+                <ValueContainer>
+                    <Text color="gray" size="sm" weight="bold">
+                        Value
+                    </Text>
+                    <Text color="gray">{propRow?.values[0]?.value}</Text>
+                </ValueContainer>
+                {relatedDescription && (
+                    <ValueContainer>
+                        <Text color="gray" size="sm" weight="bold">
+                            Description
+                        </Text>
+                        <Text color="gray">{relatedDescription}</Text>
+                    </ValueContainer>
+                )}
+            </TooltipContainer>
+        );
+    };
+
+    return (
+        <StyledTooltip
+            showArrow={false}
+            title={<BadgeTooltip />}
+            color={colors.white}
+            overlayInnerStyle={{ width: 250, padding: 16 }}
+        >
+            <BadgeContainer>
+                <Pill
+                    label={propRow?.values[0]?.value?.toString() || ''}
+                    size="sm"
+                    colorScheme="violet"
+                    clickable={false}
+                />
+            </BadgeContainer>
+        </StyledTooltip>
+    );
+};
+
+export default StructuredPropertyBadge;
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/utils.ts b/datahub-web-react/src/app/entity/shared/containers/profile/header/utils.ts
new file mode 100644
index 0000000000000..6ab469725b51a
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/utils.ts
@@ -0,0 +1,29 @@
+import EntityRegistry from '@src/app/entity/EntityRegistry';
+import { EntityType, StructuredPropertiesEntry } from '../../../../../../types.generated';
+import { capitalizeFirstLetterOnly } from '../../../../../shared/textUtil';
+import { GenericEntityProperties } from '../../../types';
+
+export function getDisplayedEntityType(
+    entityData: GenericEntityProperties | null,
+    entityRegistry: EntityRegistry,
+    entityType: EntityType,
+) {
+    return (
+        entityData?.entityTypeOverride ||
+        capitalizeFirstLetterOnly(entityData?.subTypes?.typeNames?.[0]) ||
+        entityRegistry.getEntityName(entityType) ||
+        ''
+    );
+}
+
+export function getEntityPlatforms(entityType: EntityType | null, entityData: GenericEntityProperties | null) {
+    const platform = entityType === EntityType.SchemaField ? entityData?.parent?.platform : entityData?.platform;
+    const platforms =
+        entityType === EntityType.SchemaField ? entityData?.parent?.siblingPlatforms : entityData?.siblingPlatforms;
+
+    return { platform, platforms };
+}
+
+export function filterForAssetBadge(prop: StructuredPropertiesEntry) {
+    return prop.structuredProperty.settings?.showAsAssetBadge && !prop.structuredProperty.settings?.isHidden;
+}
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePathV2.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePathV2.tsx
index d91bb565c0cd1..7805c8c73b357 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePathV2.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePathV2.tsx
@@ -87,7 +87,7 @@ export default function ProfileNavBrowsePathV2({ urn, type }: Props) {
                         {entityRegistry.getDisplayName(EntityType.DataPlatform, entityData.platform)}
                     </BreadcrumbItem>
                 )}
-                {entityData?.browsePathV2?.path.map((pathEntry, index) => (
+                {entityData?.browsePathV2?.path?.map((pathEntry, index) => (
                     <BreadcrumbItem
                         key={pathEntry.name}
                         disabled={!isBrowsable}
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx
index 681f89831b92c..818d75c37696d 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx
@@ -5,6 +5,8 @@ import { useGetSearchResultsLazyQuery } from '../../../../../../../graphql/searc
 import { Container, Entity, EntityType } from '../../../../../../../types.generated';
 import { useEnterKeyListener } from '../../../../../../shared/useEnterKeyListener';
 import { useEntityRegistry } from '../../../../../../useEntityRegistry';
+import { getParentEntities } from '../../../../../../search/filters/utils';
+import ParentEntities from '../../../../../../search/filters/ParentEntities';
 
 type Props = {
     onCloseModal: () => void;
@@ -26,7 +28,7 @@ const StyleTag = styled(Tag)`
     align-items: center;
 `;
 
-const PreviewImage = styled.img`
+export const PreviewImage = styled.img`
     max-height: 18px;
     width: auto;
     object-fit: contain;
@@ -34,6 +36,10 @@ const PreviewImage = styled.img`
     margin-right: 4px;
 `;
 
+export const ParentWrapper = styled.div`
+    max-width: 400px;
+`;
+
 export const ContainerSelectModal = ({ onCloseModal, defaultValues, onOkOverride, titleOverride }: Props) => {
     const [containerSearch, { data: platforSearchData }] = useGetSearchResultsLazyQuery();
     const entityRegistry = useEntityRegistry();
@@ -65,10 +71,16 @@ export const ContainerSelectModal = ({ onCloseModal, defaultValues, onOkOverride
     // Renders a search result in the select dropdown.
     const renderSearchResult = (entity: Container) => {
         const displayName = entityRegistry.getDisplayName(EntityType.Container, entity);
+        const parentEntities: Entity[] = getParentEntities(entity as Entity) || [];
 
         const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName;
         return (
             <Tooltip title={displayName}>
+                {parentEntities.length > 0 && (
+                    <ParentWrapper>
+                        <ParentEntities parentEntities={parentEntities} />
+                    </ParentWrapper>
+                )}
                 <PreviewImage src={entity.platform?.properties?.logoUrl || undefined} alt={entity.properties?.name} />
                 <span>{truncatedDisplayName}</span>
             </Tooltip>
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/DataProduct/DataProductSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/DataProduct/DataProductSection.tsx
index 1d489e88b5050..ec65b31968d83 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/DataProduct/DataProductSection.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/DataProduct/DataProductSection.tsx
@@ -67,7 +67,7 @@ export default function DataProductSection({ readOnly }: Props) {
     };
 
     return (
-        <>
+        <div>
             <SidebarHeader title="Data Product" />
             {dataProduct && (
                 <DataProductLink
@@ -101,6 +101,6 @@ export default function DataProductSection({ readOnly }: Props) {
                     setDataProduct={setDataProduct}
                 />
             )}
-        </>
+        </div>
     );
 }
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/DataProduct/SetDataProductModal.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/DataProduct/SetDataProductModal.tsx
index 904b4c064f001..ece50e5ae1496 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/DataProduct/SetDataProductModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/DataProduct/SetDataProductModal.tsx
@@ -146,7 +146,7 @@ export default function SetDataProductModal({
                 tagRender={tagRender}
                 onBlur={() => setQuery('')}
             >
-                {data?.searchAcrossEntities?.searchResults.map((result) => (
+                {data?.searchAcrossEntities?.searchResults?.map((result) => (
                     <Select.Option value={result.entity.urn} key={result.entity.urn}>
                         <OptionWrapper>
                             {entityRegistry.getIcon(EntityType.DataProduct, 12, IconStyleType.ACCENT, 'black')}
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/FormInfo/useGetPromptInfo.ts b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/FormInfo/useGetPromptInfo.ts
index 252cefb9f6f4f..e141114f2c478 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/FormInfo/useGetPromptInfo.ts
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/FormInfo/useGetPromptInfo.ts
@@ -18,10 +18,10 @@ export default function useGetPromptInfo(formUrn?: string) {
         requiredEntityPrompts,
     } = getEntityPromptsInfo(prompts, entityData);
     const { fieldPrompts, numRequiredFieldPromptsRemaining, numOptionalFieldPromptsRemaining, requiredFieldPrompts } =
-        getFieldPromptsInfo(prompts, entityData, entityWithSchema?.schemaMetadata?.fields.length || 0, formUrn);
+        getFieldPromptsInfo(prompts, entityData, entityWithSchema?.schemaMetadata?.fields?.length || 0, formUrn);
     // Multiply number of field prompts by number of schema fields for total number of schema field prompts
     const totalRequiredSchemaFieldPrompts =
-        (entityWithSchema?.schemaMetadata?.fields.length || 0) * requiredFieldPrompts.length;
+        (entityWithSchema?.schemaMetadata?.fields?.length || 0) * requiredFieldPrompts.length;
 
     const numRequiredPromptsRemaining = numRequiredEntityPromptsRemaining + numRequiredFieldPromptsRemaining;
     const numOptionalPromptsRemaining = numOptionalEntityPromptsRemaining + numOptionalFieldPromptsRemaining;
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/FormInfo/utils.ts b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/FormInfo/utils.ts
index 4a0ae2131623f..c7c0e6f48b411 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/FormInfo/utils.ts
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/FormInfo/utils.ts
@@ -197,7 +197,7 @@ export function findPromptAssociation(prompt: FormPrompt, allPrompts: Array<Form
 // Get the prompts for a given form
 export function getPromptsForForm(formUrn: string, entityData: GenericEntityProperties | null) {
     const formAssociation = getFormAssociation(formUrn, entityData);
-    return formAssociation?.form.info.prompts || [];
+    return formAssociation?.form?.info?.prompts || [];
 }
 
 /*
@@ -279,7 +279,7 @@ export function isVerificationComplete(entityData: GenericEntityProperties | nul
 
 export function isFormVerificationType(entityData: GenericEntityProperties | null, formUrn: string) {
     const formAssociation = getFormAssociation(formUrn, entityData);
-    return formAssociation?.form.info.type === FormType.Verification;
+    return formAssociation?.form?.info?.type === FormType.Verification;
 }
 
 /*
@@ -317,6 +317,6 @@ export function getVerificationAuditStamp(entityData: GenericEntityProperties |
 export function getBulkByQuestionPrompts(formUrn: string, entityData: GenericEntityProperties | null) {
     const formAssociation = getFormAssociation(formUrn, entityData);
     return (
-        formAssociation?.form.info.prompts.filter((prompt) => !SCHEMA_FIELD_PROMPT_TYPES.includes(prompt.type)) || []
+        formAssociation?.form?.info?.prompts?.filter((prompt) => !SCHEMA_FIELD_PROMPT_TYPES.includes(prompt.type)) || []
     );
 }
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx
index 0ce3c9641d559..c3896baedace7 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx
@@ -29,7 +29,7 @@ export default function LinkButton({ link }: Props) {
             href={link.url}
             target="_blank"
             rel="noreferrer"
-            key={`${link.label}-${link.url}-${link.author}`}
+            key={`${link.label}-${link.url}-${link.actor.urn}`}
         >
             <LinkOutlined />
             {link.description || link.label}
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarHeader.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarHeader.tsx
index 0ee3fcb90e575..a5c7cce93a42a 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarHeader.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarHeader.tsx
@@ -15,14 +15,15 @@ const HeaderContainer = styled.div`
 
 type Props = {
     title: string;
+    titleComponent?: React.ReactNode;
     actions?: React.ReactNode;
     children?: React.ReactNode;
 };
 
-export const SidebarHeader = ({ title, actions, children }: Props) => {
+export const SidebarHeader = ({ title, titleComponent, actions, children }: Props) => {
     return (
         <HeaderContainer>
-            <Typography.Title level={5}>{title}</Typography.Title>
+            {titleComponent || <Typography.Title level={5}>{title}</Typography.Title>}
             {actions && <div>{actions}</div>}
             {children}
         </HeaderContainer>
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection.tsx
new file mode 100644
index 0000000000000..41e07d49b2543
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection.tsx
@@ -0,0 +1,146 @@
+import { useUserContext } from '@src/app/context/useUserContext';
+import { useEntityData } from '@src/app/entity/shared/EntityContext';
+import { StyledList } from '@src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties';
+import { EditColumn } from '@src/app/entity/shared/tabs/Properties/Edit/EditColumn';
+import StructuredPropertyValue from '@src/app/entity/shared/tabs/Properties/StructuredPropertyValue';
+import { PropertyRow } from '@src/app/entity/shared/tabs/Properties/types';
+import EmptySectionText from '@src/app/shared/sidebar/EmptySectionText';
+import {
+    getDisplayName,
+    getEntityTypesPropertyFilter,
+    getNotHiddenPropertyFilter,
+    getPropertyRowFromSearchResult,
+} from '@src/app/govern/structuredProperties/utils';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { useGetSearchResultsForMultipleQuery } from '@src/graphql/search.generated';
+import { EntityType, SchemaField, SearchResult, StdDataType, StructuredPropertyEntity } from '@src/types.generated';
+import {
+    SHOW_IN_ASSET_SUMMARY_PROPERTY_FILTER_NAME,
+    SHOW_IN_COLUMNS_TABLE_PROPERTY_FILTER_NAME,
+} from '@src/app/search/utils/constants';
+import {
+    SectionHeader,
+    StyledDivider,
+} from '@src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/components';
+import { useGetEntityWithSchema } from '@src/app/entity/shared/tabs/Dataset/Schema/useGetEntitySchema';
+import React from 'react';
+import { SidebarHeader } from '../SidebarHeader';
+
+interface Props {
+    properties?: {
+        schemaField?: SchemaField;
+        schemaColumnProperties?: SearchResult[];
+    };
+}
+
+const SidebarStructuredPropsSection = ({ properties }: Props) => {
+    const schemaField = properties?.schemaField;
+    const schemaColumnProperties = properties?.schemaColumnProperties;
+    const { entityData, entityType } = useEntityData();
+    const me = useUserContext();
+    const entityRegistry = useEntityRegistry();
+    const { refetch: refetchSchema } = useGetEntityWithSchema(true);
+
+    const currentProperties = schemaField
+        ? schemaField?.schemaFieldEntity?.structuredProperties
+        : entityData?.structuredProperties;
+
+    const inputs = {
+        types: [EntityType.StructuredProperty],
+        query: '',
+        start: 0,
+        count: 50,
+        searchFlags: { skipCache: true },
+        orFilters: [
+            {
+                and: [
+                    getEntityTypesPropertyFilter(entityRegistry, !!schemaField, entityType),
+                    getNotHiddenPropertyFilter(),
+                    {
+                        field: schemaField
+                            ? SHOW_IN_COLUMNS_TABLE_PROPERTY_FILTER_NAME
+                            : SHOW_IN_ASSET_SUMMARY_PROPERTY_FILTER_NAME,
+                        values: ['true'],
+                    },
+                ],
+            },
+        ],
+    };
+
+    // Execute search
+    const { data } = useGetSearchResultsForMultipleQuery({
+        variables: {
+            input: inputs,
+        },
+        skip: !!schemaColumnProperties,
+        fetchPolicy: 'cache-first',
+    });
+
+    const entityTypeProperties = schemaColumnProperties || data?.searchAcrossEntities?.searchResults;
+
+    const canEditProperties = me.platformPrivileges?.manageStructuredProperties;
+
+    return (
+        <>
+            {entityTypeProperties?.map((property) => {
+                const propertyRow: PropertyRow | undefined = getPropertyRowFromSearchResult(
+                    property,
+                    currentProperties,
+                );
+                const isRichText = propertyRow?.dataType?.info?.type === StdDataType.RichText;
+                const values = propertyRow?.values;
+                const hasMultipleValues = values && values.length > 1;
+                const propertyName = getDisplayName(property.entity as StructuredPropertyEntity);
+
+                return (
+                    <>
+                        <div>
+                            <SidebarHeader
+                                title={propertyName}
+                                titleComponent={<SectionHeader>{propertyName}</SectionHeader>}
+                                actions={
+                                    canEditProperties && (
+                                        <>
+                                            <EditColumn
+                                                structuredProperty={property.entity as StructuredPropertyEntity}
+                                                values={values?.map((v) => v.value) || []}
+                                                isAddMode={!values}
+                                                associatedUrn={schemaField?.schemaFieldEntity?.urn}
+                                                refetch={schemaField ? refetchSchema : undefined}
+                                            />
+                                        </>
+                                    )
+                                }
+                            />
+
+                            {values ? (
+                                <>
+                                    {hasMultipleValues ? (
+                                        <StyledList>
+                                            {values.map((value) => (
+                                                <li>
+                                                    <StructuredPropertyValue value={value} isRichText={isRichText} />
+                                                </li>
+                                            ))}
+                                        </StyledList>
+                                    ) : (
+                                        <>
+                                            {values?.map((value) => (
+                                                <StructuredPropertyValue value={value} isRichText={isRichText} />
+                                            ))}
+                                        </>
+                                    )}
+                                </>
+                            ) : (
+                                <EmptySectionText message="No value set" />
+                            )}
+                        </div>
+                        {schemaField && <StyledDivider />}
+                    </>
+                );
+            })}
+        </>
+    );
+};
+
+export default SidebarStructuredPropsSection;
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts b/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts
index c32ba40563055..bb35755826a89 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts
@@ -231,7 +231,7 @@ export function sortEntityProfileTabs(appConfig: AppConfig, entityType: EntityTy
     const sortedTabs = [...tabs];
 
     if (entityType === EntityType.Domain && appConfig.visualConfig.entityProfiles?.domain?.defaultTab) {
-        const defaultTabId = appConfig.visualConfig.entityProfiles?.domain.defaultTab;
+        const defaultTabId = appConfig.visualConfig.entityProfiles?.domain?.defaultTab;
         sortTabsWithDefaultTabId(sortedTabs, defaultTabId);
     }
 
diff --git a/datahub-web-react/src/app/entity/shared/embed/UpstreamHealth/FailingAssertions.tsx b/datahub-web-react/src/app/entity/shared/embed/UpstreamHealth/FailingAssertions.tsx
index 0784be9ff2f88..939418ffc1104 100644
--- a/datahub-web-react/src/app/entity/shared/embed/UpstreamHealth/FailingAssertions.tsx
+++ b/datahub-web-react/src/app/entity/shared/embed/UpstreamHealth/FailingAssertions.tsx
@@ -28,7 +28,7 @@ export default function FailingAssertions({ upstreamSummary }: Props) {
             failing assertions
             <FailingDataWrapper>
                 {datasetsWithFailingAssertions.map((dataset) => {
-                    const totalNumAssertions = dataset.assertions?.assertions.length;
+                    const totalNumAssertions = dataset.assertions?.assertions?.length;
                     const numAssertionsFailing = getNumAssertionsFailing(dataset);
 
                     return (
diff --git a/datahub-web-react/src/app/entity/shared/embed/UpstreamHealth/utils.ts b/datahub-web-react/src/app/entity/shared/embed/UpstreamHealth/utils.ts
index 374c00f34a8a8..84b18725972e6 100644
--- a/datahub-web-react/src/app/entity/shared/embed/UpstreamHealth/utils.ts
+++ b/datahub-web-react/src/app/entity/shared/embed/UpstreamHealth/utils.ts
@@ -53,7 +53,7 @@ export function extractUpstreamSummary(upstreams: Entity[]) {
 export function getNumAssertionsFailing(dataset: Dataset) {
     let numFailing = 0;
 
-    dataset.assertions?.assertions.forEach((assertion) => {
+    dataset.assertions?.assertions?.forEach((assertion) => {
         if (assertion.runEvents?.failed) {
             numFailing += 1;
         }
diff --git a/datahub-web-react/src/app/entity/shared/entityForm/EntityFormContextProvider.tsx b/datahub-web-react/src/app/entity/shared/entityForm/EntityFormContextProvider.tsx
index 278cb02a89ac9..44d771515f690 100644
--- a/datahub-web-react/src/app/entity/shared/entityForm/EntityFormContextProvider.tsx
+++ b/datahub-web-react/src/app/entity/shared/entityForm/EntityFormContextProvider.tsx
@@ -19,7 +19,7 @@ export default function EntityFormContextProvider({ children, formUrn }: Props)
     const { entityData, refetch: refetchEntityProfile, loading: profileLoading } = useEntityContext();
     const formAssociation = getFormAssociation(formUrn, entityData);
     const initialPromptId =
-        formAssociation?.form.info.prompts.filter((prompt) => !SCHEMA_FIELD_PROMPT_TYPES.includes(prompt.type))[0]
+        formAssociation?.form?.info?.prompts?.filter((prompt) => !SCHEMA_FIELD_PROMPT_TYPES.includes(prompt.type))[0]
             ?.id || null;
     const isVerificationType = isFormVerificationType(entityData, formUrn);
     const [formView, setFormView] = useState<FormView>(FormView.BY_ENTITY);
diff --git a/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx b/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx
index 88b3c25162ec5..b5f3cd770078a 100644
--- a/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx
+++ b/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx
@@ -54,10 +54,10 @@ function Form({ formUrn }: Props) {
     if (!hasRendered) return <Loading />;
 
     const formAssociation = getFormAssociation(formUrn, entityData);
-    const title = formAssociation?.form.info.name;
+    const title = formAssociation?.form?.info?.name;
     const associatedUrn = formAssociation?.associatedUrn;
-    const description = formAssociation?.form.info.description;
-    const owners = formAssociation?.form.ownership?.owners;
+    const description = formAssociation?.form?.info?.description;
+    const owners = formAssociation?.form?.ownership?.owners;
 
     return (
         <TabWrapper>
diff --git a/datahub-web-react/src/app/entity/shared/entityForm/prompts/StructuredPropertyPrompt/utils.ts b/datahub-web-react/src/app/entity/shared/entityForm/prompts/StructuredPropertyPrompt/utils.ts
index 1050c5fcde728..9840f9a2cbd05 100644
--- a/datahub-web-react/src/app/entity/shared/entityForm/prompts/StructuredPropertyPrompt/utils.ts
+++ b/datahub-web-react/src/app/entity/shared/entityForm/prompts/StructuredPropertyPrompt/utils.ts
@@ -20,7 +20,7 @@ export function getInitialValues(prompt: FormPrompt, entityData: GenericEntityPr
             (propAssignment) => propAssignment.structuredProperty.urn === structuredProperty?.urn,
         );
     }
-    return structuredPropertyAssignment?.values.map((value) => getStructuredPropertyValue(value as PropertyValue));
+    return structuredPropertyAssignment?.values?.map((value) => getStructuredPropertyValue(value as PropertyValue));
 }
 
 export function getInitialEntitiesForUrnPrompt(
diff --git a/datahub-web-react/src/app/entity/shared/entityForm/schemaFieldPrompts/SchemaFieldPrompts.tsx b/datahub-web-react/src/app/entity/shared/entityForm/schemaFieldPrompts/SchemaFieldPrompts.tsx
index 087a42e3f8000..0a39d2ef8a5f4 100644
--- a/datahub-web-react/src/app/entity/shared/entityForm/schemaFieldPrompts/SchemaFieldPrompts.tsx
+++ b/datahub-web-react/src/app/entity/shared/entityForm/schemaFieldPrompts/SchemaFieldPrompts.tsx
@@ -26,7 +26,7 @@ export default function SchemaFieldPrompts({ prompts, associatedUrn }: Props) {
         <>
             <Divider />
             <FieldPromptsTitle data-testid="field-level-requirements">Field-Level Requirements</FieldPromptsTitle>
-            {entityWithSchema?.schemaMetadata?.fields.map((field) => (
+            {entityWithSchema?.schemaMetadata?.fields?.map((field) => (
                 <VirtualScrollChild key={field.fieldPath} height={50} triggerOnce>
                     <SchemaFieldDropdown prompts={prompts} field={field as SchemaField} associatedUrn={associatedUrn} />
                 </VirtualScrollChild>
diff --git a/datahub-web-react/src/app/entity/shared/entityForm/useShouldShowVerificationPrompt.ts b/datahub-web-react/src/app/entity/shared/entityForm/useShouldShowVerificationPrompt.ts
index d7a8a417a0c86..8987848dc12dd 100644
--- a/datahub-web-react/src/app/entity/shared/entityForm/useShouldShowVerificationPrompt.ts
+++ b/datahub-web-react/src/app/entity/shared/entityForm/useShouldShowVerificationPrompt.ts
@@ -28,7 +28,7 @@ export default function useShouldShowVerificationPrompt(formUrn: string) {
     const { entityData } = useEntityData();
     const formVerification = getFormVerification(formUrn, entityData);
     const formAssociation = getFormAssociation(formUrn, entityData);
-    const formType = formAssociation?.form.info.type;
+    const formType = formAssociation?.form?.info?.type;
 
     return shouldShowVerificationPrompt({
         formType,
diff --git a/datahub-web-react/src/app/entity/shared/siblingUtils.ts b/datahub-web-react/src/app/entity/shared/siblingUtils.ts
index 2f50dc99df191..aa9e4bcb5e46e 100644
--- a/datahub-web-react/src/app/entity/shared/siblingUtils.ts
+++ b/datahub-web-react/src/app/entity/shared/siblingUtils.ts
@@ -5,6 +5,7 @@ import * as QueryString from 'query-string';
 import { Dataset, Entity, Maybe, SiblingProperties } from '../../../types.generated';
 import { GenericEntityProperties } from './types';
 import { useIsShowSeparateSiblingsEnabled } from '../../useAppConfig';
+import { downgradeV2FieldPath } from '../dataset/profile/schema/utils/utils';
 
 export function stripSiblingsFromEntity(entity: any) {
     return {
@@ -55,16 +56,30 @@ const combineMerge = (target, source, options) => {
     return destination;
 };
 
-function convertObjectKeysToLowercase(object: Record<string, unknown>) {
-    return Object.fromEntries(Object.entries(object).map(([key, value]) => [key.toLowerCase(), value]));
+// this function is responsible for normalizing object keys to make sure merging on key matches keys appropriately
+function normalizeObjectKeys(object: Record<string, unknown>, isSchemaField = false) {
+    return Object.fromEntries(
+        Object.entries(object).map(([key, value]) => {
+            let normalizedKey = key.toLowerCase();
+            if (isSchemaField) {
+                normalizedKey = downgradeV2FieldPath(normalizedKey) || normalizedKey;
+            }
+            return [normalizedKey, value];
+        }),
+    );
 }
 
 // use when you want to merge an array of objects by key in the object as opposed to by index of array
-const mergeArrayOfObjectsByKey = (destinationArray: any[], sourceArray: any[], key: string) => {
-    const destination = convertObjectKeysToLowercase(keyBy(destinationArray, key));
-    const source = convertObjectKeysToLowercase(keyBy(sourceArray, key));
+const mergeArrayOfObjectsByKey = (destinationArray: any[], sourceArray: any[], key: string, isSchemaField = false) => {
+    const destination = normalizeObjectKeys(keyBy(destinationArray, key), isSchemaField);
+    const source = normalizeObjectKeys(keyBy(sourceArray, key), isSchemaField);
 
-    return values(merge(destination, source));
+    return values(
+        merge(destination, source, {
+            arrayMerge: combineMerge,
+            customMerge,
+        }),
+    );
 };
 
 const mergeTags = (destinationArray, sourceArray, _options) => {
@@ -88,7 +103,7 @@ const mergeOwners = (destinationArray, sourceArray, _options) => {
 };
 
 const mergeFields = (destinationArray, sourceArray, _options) => {
-    return mergeArrayOfObjectsByKey(destinationArray, sourceArray, 'fieldPath');
+    return mergeArrayOfObjectsByKey(destinationArray, sourceArray, 'fieldPath', true);
 };
 
 function getArrayMergeFunction(key) {
@@ -112,7 +127,7 @@ function getArrayMergeFunction(key) {
     }
 }
 
-const customMerge = (isPrimary, key) => {
+function customMerge(isPrimary, key) {
     if (key === 'upstream' || key === 'downstream') {
         return (_secondary, primary) => primary;
     }
@@ -145,7 +160,7 @@ const customMerge = (isPrimary, key) => {
             customMerge: customMerge.bind({}, isPrimary),
         });
     };
-};
+}
 
 export const getEntitySiblingData = <T>(baseEntity: T): Maybe<SiblingProperties> => {
     if (!baseEntity) {
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx
index 5de532d17a676..f5789447d953d 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx
@@ -3,8 +3,8 @@ import styled from 'styled-components';
 import { Button, Table } from 'antd';
 import { SpinProps } from 'antd/es/spin';
 import { LoadingOutlined } from '@ant-design/icons';
-import { useBaseEntity } from '../../../EntityContext';
-import { GetDatasetQuery, useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated';
+import { useEntityData } from '../../../EntityContext';
+import { useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated';
 import { handleAccessRoles } from './utils';
 import AccessManagerDescription from './AccessManagerDescription';
 
@@ -60,11 +60,12 @@ const AccessButton = styled(Button)`
 `;
 
 export default function AccessManagement() {
-    const baseEntity = useBaseEntity<GetDatasetQuery>();
+    const { entityData } = useEntityData();
+    const entityUrn = (entityData as any)?.urn;
 
     const { data: externalRoles, loading: isLoading } = useGetExternalRolesQuery({
-        variables: { urn: baseEntity?.dataset?.urn as string },
-        skip: !baseEntity?.dataset?.urn,
+        variables: { urn: entityUrn as string },
+        skip: !entityUrn,
     });
 
     const columns = [
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/utils.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/utils.tsx
index 5b61e0df5f53e..96a99b172b574 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/utils.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/utils.tsx
@@ -2,8 +2,8 @@ export function handleAccessRoles(externalRoles) {
     const accessRoles = new Array<any>();
     if (
         externalRoles?.dataset?.access &&
-        externalRoles?.dataset?.access.roles &&
-        externalRoles?.dataset?.access.roles.length > 0
+        externalRoles?.dataset?.access?.roles &&
+        externalRoles?.dataset?.access?.roles?.length > 0
     ) {
         externalRoles?.dataset?.access?.roles?.forEach((userRoles) => {
             const role = {
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/utils/filterQueries.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/utils/filterQueries.ts
index a8ec960ea2e08..fb97c8235cbe6 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/utils/filterQueries.ts
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/utils/filterQueries.ts
@@ -10,9 +10,9 @@ export const filterQueries = (filterText, queries: Query[]) => {
     const lowerFilterText = filterText.toLowerCase();
     return queries.filter((query) => {
         return (
-            query.title?.toLowerCase().includes(lowerFilterText) ||
-            query.description?.toLowerCase().includes(lowerFilterText) ||
-            query.query?.toLowerCase().includes(lowerFilterText)
+            query.title?.toLowerCase()?.includes(lowerFilterText) ||
+            query.description?.toLowerCase()?.includes(lowerFilterText) ||
+            query.query?.toLowerCase()?.includes(lowerFilterText)
         );
     });
 };
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx
index 0bfd5255f3065..e287e3675c26a 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx
@@ -27,6 +27,8 @@ import PropertiesColumn from './components/PropertiesColumn';
 import SchemaFieldDrawer from './components/SchemaFieldDrawer/SchemaFieldDrawer';
 import useBusinessAttributeRenderer from './utils/useBusinessAttributeRenderer';
 import { useBusinessAttributesFlag } from '../../../../../useAppConfig';
+import { useGetTableColumnProperties } from './useGetTableColumnProperties';
+import { useGetStructuredPropColumns } from './useGetStructuredPropColumns';
 
 const TableContainer = styled.div`
     overflow: inherit;
@@ -126,6 +128,9 @@ export default function SchemaTable({
     const schemaTitleRenderer = useSchemaTitleRenderer(schemaMetadata, setSelectedFkFieldPath, filterText);
     const schemaBlameRenderer = useSchemaBlameRenderer(schemaFieldBlameList);
 
+    const tableColumnStructuredProps = useGetTableColumnProperties();
+    const structuredPropColumns = useGetStructuredPropColumns(tableColumnStructuredProps);
+
     const fieldColumn = {
         width: '22%',
         title: 'Field',
@@ -188,7 +193,7 @@ export default function SchemaTable({
     function getCount(fieldPath: any) {
         const data: any =
             usageStats?.aggregations?.fields &&
-            usageStats?.aggregations?.fields.find((field) => {
+            usageStats?.aggregations?.fields?.find((field) => {
                 return field?.fieldName === fieldPath;
             });
         return data && data.count;
@@ -221,6 +226,10 @@ export default function SchemaTable({
         allColumns = [...allColumns, propertiesColumn];
     }
 
+    if (structuredPropColumns) {
+        allColumns = [...allColumns, ...structuredPropColumns];
+    }
+
     if (hasUsageStats) {
         allColumns = [...allColumns, usageColumn];
     }
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/ForeignKeyLabel.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/ForeignKeyLabel.tsx
index fa4826551c3a4..7aec54d9f3b87 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/ForeignKeyLabel.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/ForeignKeyLabel.tsx
@@ -38,7 +38,7 @@ export default function ForeignKeyLabel({
     const selectedFk = useContext(FkContext);
 
     const onOpenFk = () => {
-        if (selectedFk?.fieldPath.trim() === fieldPath.trim() && selectedFk?.constraint?.name === constraint?.name) {
+        if (selectedFk?.fieldPath?.trim() === fieldPath.trim() && selectedFk?.constraint?.name === constraint?.name) {
             onClick(null);
         } else {
             onClick({ fieldPath, constraint });
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PropertiesColumn.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PropertiesColumn.tsx
index b74de3e94e554..74d14cb0db753 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PropertiesColumn.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PropertiesColumn.tsx
@@ -17,7 +17,9 @@ interface Props {
 
 export default function PropertiesColumn({ field }: Props) {
     const { schemaFieldEntity } = field;
-    const numProperties = schemaFieldEntity?.structuredProperties?.properties?.length;
+    const numProperties = schemaFieldEntity?.structuredProperties?.properties?.filter(
+        (prop) => !prop.structuredProperty.settings?.isHidden,
+    )?.length;
 
     if (!schemaFieldEntity || !numProperties) return null;
 
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PropertyTypeLabel.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PropertyTypeLabel.tsx
index 366fc4762b210..b4ffce294d136 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PropertyTypeLabel.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PropertyTypeLabel.tsx
@@ -31,8 +31,8 @@ export default function PropertyTypeLabel({ type, dataType }: Props) {
     const nativeFallback = type.type === SchemaFieldDataType.Null;
 
     const typeText =
-        dataType?.info.displayName ||
-        dataType?.info.type ||
+        dataType?.info?.displayName ||
+        dataType?.info?.type ||
         (nativeFallback ? truncate(250, nativeDataType) : type.type);
 
     return <PropertyTypeBadge count={capitalizeFirstLetterOnly(typeText)} />;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties.tsx
index 9a0da20f22dfd..98cfa296508cb 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties.tsx
@@ -1,51 +1,80 @@
+import { useEntityData } from '@src/app/entity/shared/EntityContext';
 import React from 'react';
 import styled from 'styled-components';
-import { SchemaField, StdDataType } from '../../../../../../../../types.generated';
-import { SectionHeader, StyledDivider } from './components';
-import { mapStructuredPropertyValues } from '../../../../Properties/useStructuredProperties';
-import StructuredPropertyValue from '../../../../Properties/StructuredPropertyValue';
+import { SchemaField, SearchResult, StdDataType } from '../../../../../../../../types.generated';
+import AddPropertyButton from '../../../../Properties/AddPropertyButton';
 import { EditColumn } from '../../../../Properties/Edit/EditColumn';
+import StructuredPropertyValue from '../../../../Properties/StructuredPropertyValue';
+import { mapStructuredPropertyValues } from '../../../../Properties/useStructuredProperties';
 import { useGetEntityWithSchema } from '../../useGetEntitySchema';
+import { StyledDivider } from './components';
 
-const PropertyTitle = styled.div`
+export const PropertyTitle = styled.div`
     font-size: 14px;
     font-weight: 700;
     margin-bottom: 4px;
 `;
 
-const PropertyWrapper = styled.div`
+export const PropertyWrapper = styled.div`
     margin-bottom: 12px;
     display: flex;
     justify-content: space-between;
 `;
 
-const PropertiesWrapper = styled.div`
+export const PropertiesWrapper = styled.div`
     padding-left: 16px;
 `;
 
-const StyledList = styled.ul`
+export const StyledList = styled.ul`
     padding-left: 24px;
 `;
 
+export const Header = styled.div`
+    font-size: 16px;
+    font-weight: 600;
+    margin-bottom: 16px;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+`;
+
 interface Props {
     expandedField: SchemaField;
+    schemaColumnProperties?: SearchResult[];
 }
 
-export default function FieldProperties({ expandedField }: Props) {
+export default function FieldProperties({ expandedField, schemaColumnProperties }: Props) {
     const { schemaFieldEntity } = expandedField;
     const { refetch } = useGetEntityWithSchema(true);
+    const { entityData } = useEntityData();
     const properties =
-        schemaFieldEntity?.structuredProperties?.properties?.filter((prop) => prop.structuredProperty.exists) || [];
+        schemaFieldEntity?.structuredProperties?.properties?.filter(
+            (prop) =>
+                prop.structuredProperty.exists &&
+                !prop.structuredProperty.settings?.isHidden &&
+                !schemaColumnProperties?.find((p) => p.entity.urn === prop.structuredProperty.urn),
+        ) || [];
+
+    const canEditProperties =
+        entityData?.parent?.privileges?.canEditProperties || entityData?.privileges?.canEditProperties;
 
-    if (!schemaFieldEntity || !properties.length) return null;
+    if (!schemaFieldEntity) return null;
 
     return (
         <>
-            <SectionHeader>Properties</SectionHeader>
+            <Header>
+                Properties
+                <AddPropertyButton
+                    fieldUrn={schemaFieldEntity?.urn}
+                    fieldProperties={schemaFieldEntity.structuredProperties}
+                    refetch={refetch}
+                    isV1Drawer
+                />
+            </Header>
             <PropertiesWrapper>
                 {properties.map((structuredProp) => {
                     const isRichText =
-                        structuredProp.structuredProperty.definition.valueType?.info.type === StdDataType.RichText;
+                        structuredProp.structuredProperty.definition.valueType?.info?.type === StdDataType.RichText;
                     const valuesData = mapStructuredPropertyValues(structuredProp);
                     const hasMultipleValues = valuesData.length > 1;
 
@@ -71,12 +100,14 @@ export default function FieldProperties({ expandedField }: Props) {
                                     </>
                                 )}
                             </div>
-                            <EditColumn
-                                structuredProperty={structuredProp.structuredProperty}
-                                associatedUrn={schemaFieldEntity.urn}
-                                values={valuesData.map((v) => v.value) || []}
-                                refetch={refetch}
-                            />
+                            {canEditProperties && (
+                                <EditColumn
+                                    structuredProperty={structuredProp.structuredProperty}
+                                    associatedUrn={structuredProp.associatedUrn}
+                                    values={valuesData.map((v) => v.value) || []}
+                                    refetch={refetch}
+                                />
+                            )}
                         </PropertyWrapper>
                     );
                 })}
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/SchemaFieldDrawer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/SchemaFieldDrawer.tsx
index 0d9f7a98f207c..798795c19de65 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/SchemaFieldDrawer.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/SchemaFieldDrawer.tsx
@@ -1,6 +1,7 @@
 import { Drawer } from 'antd';
 import React, { useMemo } from 'react';
 import styled from 'styled-components';
+import SidebarStructuredPropsSection from '@src/app/entity/shared/containers/profile/sidebar/StructuredProperties/SidebarStructuredPropsSection';
 import DrawerHeader from './DrawerHeader';
 import FieldHeader from './FieldHeader';
 import FieldDescription from './FieldDescription';
@@ -11,6 +12,7 @@ import FieldTags from './FieldTags';
 import FieldTerms from './FieldTerms';
 import FieldProperties from './FieldProperties';
 import FieldAttribute from './FieldAttribute';
+import useGetSchemaColumnProperties from './useGetSchemaColumnProperties';
 
 const StyledDrawer = styled(Drawer)`
     position: absolute;
@@ -47,9 +49,10 @@ export default function SchemaFieldDrawer({
     );
     const expandedField =
         expandedFieldIndex !== undefined && expandedFieldIndex !== -1 ? schemaFields[expandedFieldIndex] : undefined;
-    const editableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find((candidateEditableFieldInfo) =>
+    const editableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo?.find((candidateEditableFieldInfo) =>
         pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, expandedField?.fieldPath),
     );
+    const schemaColumnProperties = useGetSchemaColumnProperties();
 
     return (
         <StyledDrawer
@@ -75,7 +78,13 @@ export default function SchemaFieldDrawer({
                         <FieldUsageStats expandedField={expandedField} />
                         <FieldTags expandedField={expandedField} editableSchemaMetadata={editableSchemaMetadata} />
                         <FieldTerms expandedField={expandedField} editableSchemaMetadata={editableSchemaMetadata} />
-                        <FieldProperties expandedField={expandedField} />
+                        <SidebarStructuredPropsSection
+                            properties={{ schemaField: expandedField, schemaColumnProperties }}
+                        />
+                        <FieldProperties
+                            expandedField={expandedField}
+                            schemaColumnProperties={schemaColumnProperties}
+                        />
                         <FieldAttribute expandedField={expandedField} />
                     </MetadataSections>
                 </>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/useGetSchemaColumnProperties.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/useGetSchemaColumnProperties.ts
new file mode 100644
index 0000000000000..ed5af588fa036
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/useGetSchemaColumnProperties.ts
@@ -0,0 +1,39 @@
+import { getEntityTypesPropertyFilter, getNotHiddenPropertyFilter } from '@src/app/govern/structuredProperties/utils';
+import { SHOW_IN_COLUMNS_TABLE_PROPERTY_FILTER_NAME } from '@src/app/search/utils/constants';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { useGetSearchResultsForMultipleQuery } from '@src/graphql/search.generated';
+import { EntityType, SearchResult } from '@src/types.generated';
+
+export default function useGetSchemaColumnProperties() {
+    const entityRegistry = useEntityRegistry();
+
+    const inputs = {
+        types: [EntityType.StructuredProperty],
+        query: '',
+        start: 0,
+        count: 50,
+        searchFlags: { skipCache: true },
+        orFilters: [
+            {
+                and: [
+                    getEntityTypesPropertyFilter(entityRegistry, true, EntityType.SchemaField),
+                    getNotHiddenPropertyFilter(),
+                    {
+                        field: SHOW_IN_COLUMNS_TABLE_PROPERTY_FILTER_NAME,
+                        values: ['true'],
+                    },
+                ],
+            },
+        ],
+    };
+
+    // Execute search
+    const { data } = useGetSearchResultsForMultipleQuery({
+        variables: {
+            input: inputs,
+        },
+        fetchPolicy: 'cache-first',
+    });
+
+    return data?.searchAcrossEntities?.searchResults || ([] as SearchResult[]);
+}
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/useGetStructuredPropColumns.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/useGetStructuredPropColumns.tsx
new file mode 100644
index 0000000000000..eed3fd510724b
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/useGetStructuredPropColumns.tsx
@@ -0,0 +1,22 @@
+import StructuredPropValues from '@src/app/entity/dataset/profile/schema/components/StructuredPropValues';
+import { getDisplayName } from '@src/app/govern/structuredProperties/utils';
+import { SearchResult, StructuredPropertyEntity } from '@src/types.generated';
+import React, { useMemo } from 'react';
+
+export const useGetStructuredPropColumns = (properties: SearchResult[] | undefined) => {
+    const columns = useMemo(() => {
+        return properties?.map((prop) => {
+            const name = getDisplayName(prop.entity as StructuredPropertyEntity);
+            return {
+                width: 120,
+                title: name,
+                dataIndex: 'schemaFieldEntity',
+                key: prop.entity.urn,
+                render: (record) => <StructuredPropValues schemaFieldEntity={record} propColumn={prop} />,
+                ellipsis: true,
+            };
+        });
+    }, [properties]);
+
+    return columns;
+};
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/useGetTableColumnProperties.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/useGetTableColumnProperties.ts
new file mode 100644
index 0000000000000..96ff90921b937
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/useGetTableColumnProperties.ts
@@ -0,0 +1,39 @@
+import {
+    getEntityTypesPropertyFilter,
+    getNotHiddenPropertyFilter,
+    getShowInColumnsTablePropertyFilter,
+} from '@src/app/govern/structuredProperties/utils';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { useGetSearchResultsForMultipleQuery } from '@src/graphql/search.generated';
+import { EntityType } from '@src/types.generated';
+
+export const useGetTableColumnProperties = () => {
+    const entityRegistry = useEntityRegistry();
+
+    const inputs = {
+        types: [EntityType.StructuredProperty],
+        query: '',
+        start: 0,
+        count: 50,
+        searchFlags: { skipCache: true },
+        orFilters: [
+            {
+                and: [
+                    getEntityTypesPropertyFilter(entityRegistry, true),
+                    getNotHiddenPropertyFilter(),
+                    getShowInColumnsTablePropertyFilter(),
+                ],
+            },
+        ],
+    };
+
+    // Execute search
+    const { data } = useGetSearchResultsForMultipleQuery({
+        variables: {
+            input: inputs,
+        },
+        fetchPolicy: 'cache-first',
+    });
+
+    return data?.searchAcrossEntities?.searchResults;
+};
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/filterSchemaRows.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/filterSchemaRows.ts
index 53b76d53f886a..9c0813fc2b85a 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/filterSchemaRows.ts
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/filterSchemaRows.ts
@@ -12,7 +12,7 @@ function matchesTagsOrTermsOrDescription(field: SchemaField, filterText: string,
                 .toLocaleLowerCase()
                 .includes(filterText),
         ) ||
-        field.description?.toLocaleLowerCase().includes(filterText)
+        field.description?.toLocaleLowerCase()?.includes(filterText)
     );
 }
 
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx
index bb70c2cb49303..b754624446cef 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx
@@ -22,7 +22,7 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS
     };
 
     return (description: string, record: SchemaField, index: number): JSX.Element => {
-        const editableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find((candidateEditableFieldInfo) =>
+        const editableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo?.find((candidateEditableFieldInfo) =>
             pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath),
         );
         const { schemaFieldEntity } = record;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx
index 8652f4bb2e8b8..e5f27a5001eb5 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx
@@ -21,7 +21,7 @@ export default function useTagsAndTermsRenderer(
     };
 
     const tagAndTermRender = (tags: GlobalTags, record: SchemaField) => {
-        const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find(
+        const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo?.find(
             (candidateEditableFieldInfo) => pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath),
         );
 
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRendererFeatureTable.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRendererFeatureTable.tsx
index df9e01ac4d444..ac3e13dabc86d 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRendererFeatureTable.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRendererFeatureTable.tsx
@@ -14,7 +14,7 @@ export default function useTagsAndTermsRendererFeatureTable(
     const refetch = useRefetch();
 
     const tagAndTermRender = (tags: GlobalTags, record: SchemaField, rowIndex: number | undefined) => {
-        const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find(
+        const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo?.find(
             (candidateEditableFieldInfo) => pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath),
         );
 
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDetails.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDetails.tsx
index 73259b0ec832a..ad2c4e05d7709 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDetails.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDetails.tsx
@@ -97,7 +97,7 @@ export const DatasetAssertionDetails = ({ urn, lastEvaluatedAtMillis }: Props) =
     };
 
     const completeAssertionRunEvents =
-        data?.assertion?.runEvents?.runEvents.filter((runEvent) => runEvent.status === AssertionRunStatus.Complete) ||
+        data?.assertion?.runEvents?.runEvents?.filter((runEvent) => runEvent.status === AssertionRunStatus.Complete) ||
         [];
 
     /**
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx
index 5009587c0d277..3659e527a872b 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx
@@ -29,7 +29,7 @@ export const FreshnessScheduleSummary = ({ definition, evaluationSchedule }: Pro
         case FreshnessAssertionScheduleType.FixedInterval:
             scheduleText = `In the past ${
                 definition.fixedInterval?.multiple
-            } ${definition.fixedInterval?.unit.toLocaleLowerCase()}s${
+            } ${definition.fixedInterval?.unit?.toLocaleLowerCase()}s${
                 cronStr ? `, as of ${cronstrue.toString(cronStr).toLowerCase()}` : ''
             }`;
             break;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts
index da2ae66d1ec9c..7eeb526d2bf4d 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts
@@ -9,12 +9,12 @@ export const createBuilderState = (contract?: DataContract | null): DataContract
         return {
             schema:
                 (contract?.properties?.schema?.length && {
-                    assertionUrn: contract?.properties?.schema[0].assertion.urn,
+                    assertionUrn: contract?.properties?.schema[0]?.assertion?.urn,
                 }) ||
                 undefined,
             freshness:
                 (contract?.properties?.freshness?.length && {
-                    assertionUrn: contract?.properties?.freshness[0].assertion.urn,
+                    assertionUrn: contract?.properties?.freshness[0]?.assertion?.urn,
                 }) ||
                 undefined,
             dataQuality:
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/CompactMarkdownViewer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/CompactMarkdownViewer.tsx
new file mode 100644
index 0000000000000..df93d51e721bb
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/CompactMarkdownViewer.tsx
@@ -0,0 +1,161 @@
+import { Button } from 'antd';
+import React, { useCallback, useEffect, useState } from 'react';
+import styled from 'styled-components';
+import { Editor } from './editor/Editor';
+
+const LINE_HEIGHT = 1.5;
+
+const ShowMoreWrapper = styled.div`
+    align-items: start;
+    display: flex;
+    flex-direction: column;
+`;
+
+const MarkdownContainer = styled.div<{ lineLimit?: number | null }>`
+    max-width: 100%;
+    position: relative;
+    ${(props) =>
+        props.lineLimit &&
+        props.lineLimit <= 1 &&
+        ` 
+        display: flex;
+        align-items: center;
+        gap: 4px;
+        ${ShowMoreWrapper}{
+            flex-direction: row;
+            align-items: center;
+            gap: 4px;
+        }
+    `}
+`;
+
+const CustomButton = styled(Button)`
+    padding: 0;
+    color: #676b75;
+`;
+
+const MarkdownViewContainer = styled.div<{ scrollableY: boolean }>`
+    display: block;
+    overflow-wrap: break-word;
+    word-wrap: break-word;
+    overflow-x: hidden;
+    overflow-y: ${(props) => (props.scrollableY ? 'auto' : 'hidden')};
+`;
+
+const CompactEditor = styled(Editor)<{ limit: number | null; customStyle?: React.CSSProperties }>`
+    .remirror-editor.ProseMirror {
+        ${({ limit }) => limit && `max-height: ${limit * LINE_HEIGHT}em;`}
+        h1 {
+            font-size: 1.4em;
+        }
+
+        h2 {
+            font-size: 1.3em;
+        }
+
+        h3 {
+            font-size: 1.2em;
+        }
+
+        h4 {
+            font-size: 1.1em;
+        }
+
+        h5,
+        h6 {
+            font-size: 1em;
+        }
+
+        p {
+            ${(props) => props?.customStyle?.fontSize && `font-size: ${props?.customStyle?.fontSize}`};
+            margin-bottom: 0;
+        }
+
+        padding: 0;
+    }
+`;
+
+const FixedLineHeightEditor = styled(CompactEditor)<{ customStyle?: React.CSSProperties }>`
+    .remirror-editor.ProseMirror {
+        * {
+            line-height: ${LINE_HEIGHT};
+            font-size: 1em !important;
+            margin-top: 0;
+            margin-bottom: 0;
+        }
+        p {
+            font-size: ${(props) => (props?.customStyle?.fontSize ? props?.customStyle?.fontSize : '1em')} !important;
+        }
+    }
+`;
+
+export type Props = {
+    content: string;
+    lineLimit?: number | null;
+    fixedLineHeight?: boolean;
+    isShowMoreEnabled?: boolean;
+    customStyle?: React.CSSProperties;
+    scrollableY?: boolean; // Whether the viewer is vertically scrollable.
+    handleShowMore?: () => void;
+    hideShowMore?: boolean;
+};
+
+export default function CompactMarkdownViewer({
+    content,
+    lineLimit = 4,
+    fixedLineHeight = false,
+    isShowMoreEnabled = false,
+    customStyle = {},
+    scrollableY = true,
+    handleShowMore,
+    hideShowMore,
+}: Props) {
+    const [isShowingMore, setIsShowingMore] = useState(false);
+    const [isTruncated, setIsTruncated] = useState(false);
+
+    useEffect(() => {
+        if (isShowMoreEnabled) {
+            setIsShowingMore(isShowMoreEnabled);
+        }
+        return () => {
+            setIsShowingMore(false);
+        };
+    }, [isShowMoreEnabled]);
+
+    const measuredRef = useCallback((node: HTMLDivElement | null) => {
+        if (node !== null) {
+            const resizeObserver = new ResizeObserver(() => {
+                setIsTruncated(node.scrollHeight > node.clientHeight + 1);
+            });
+            resizeObserver.observe(node);
+        }
+    }, []);
+
+    const StyledEditor = fixedLineHeight ? FixedLineHeightEditor : CompactEditor;
+
+    return (
+        <MarkdownContainer lineLimit={lineLimit}>
+            <MarkdownViewContainer scrollableY={scrollableY} ref={measuredRef}>
+                <StyledEditor
+                    customStyle={customStyle}
+                    limit={isShowingMore ? null : lineLimit}
+                    content={content}
+                    readOnly
+                />
+            </MarkdownViewContainer>
+            {hideShowMore && <>...</>}
+
+            {!hideShowMore &&
+                (isShowingMore || isTruncated) && ( // "show more" when isTruncated, "show less" when isShowingMore
+                    <ShowMoreWrapper>
+                        <CustomButton
+                            type="link"
+                            onClick={() => (handleShowMore ? handleShowMore() : setIsShowingMore(!isShowingMore))}
+                        >
+                            {isShowingMore ? 'show less' : 'show more'}
+                        </CustomButton>
+                    </ShowMoreWrapper>
+                )}
+        </MarkdownContainer>
+    );
+}
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx
index 7212198bbf61c..6eb680785599e 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx
@@ -3,7 +3,7 @@ import { Link } from 'react-router-dom';
 import styled from 'styled-components/macro';
 import { message, Button, List, Typography, Modal, Form, Input } from 'antd';
 import { LinkOutlined, DeleteOutlined, EditOutlined } from '@ant-design/icons';
-import { EntityType, InstitutionalMemoryMetadata } from '../../../../../../types.generated';
+import { InstitutionalMemoryMetadata } from '../../../../../../types.generated';
 import { useEntityData, useMutationUrn } from '../../../EntityContext';
 import { useEntityRegistry } from '../../../../../useEntityRegistry';
 import { ANTD_GRAY } from '../../../constants';
@@ -182,10 +182,8 @@ export const LinkList = ({ refetch }: LinkListProps) => {
                                 description={
                                     <>
                                         Added {formatDateString(link.created.time)} by{' '}
-                                        <Link
-                                            to={`${entityRegistry.getEntityUrl(EntityType.CorpUser, link.author.urn)}`}
-                                        >
-                                            {link.author.username}
+                                        <Link to={`${entityRegistry.getEntityUrl(link.actor.type, link.actor.urn)}`}>
+                                            {entityRegistry.getDisplayName(link.actor.type, link.actor)}
                                         </Link>
                                     </>
                                 }
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx
index fe2a8c51f9377..8ee0f637094d6 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx
@@ -50,26 +50,26 @@ export const Editor = forwardRef((props: EditorProps, ref) => {
     const { manager, state, getContext } = useRemirror({
         extensions: () => [
             new BlockquoteExtension(),
-            new BoldExtension(),
-            new BulletListExtension(),
+            new BoldExtension({}),
+            new BulletListExtension({}),
             new CodeBlockExtension({ syntaxTheme: 'base16_ateliersulphurpool_light' }),
             new CodeExtension(),
-            new DataHubMentionsExtension(),
-            new DropCursorExtension(),
+            new DataHubMentionsExtension({}),
+            new DropCursorExtension({}),
             new HardBreakExtension(),
-            new HeadingExtension(),
-            new HistoryExtension(),
-            new HorizontalRuleExtension(),
+            new HeadingExtension({}),
+            new HistoryExtension({}),
+            new HorizontalRuleExtension({}),
             new ImageExtension({ enableResizing: !readOnly }),
             new ItalicExtension(),
             new LinkExtension({ autoLink: true, defaultTarget: '_blank' }),
-            new ListItemExtension(),
+            new ListItemExtension({}),
             new MarkdownExtension({ htmlSanitizer: DOMPurify.sanitize, htmlToMarkdown, markdownToHtml }),
             new OrderedListExtension(),
             new UnderlineExtension(),
             new StrikeExtension(),
             new TableExtension({ resizable: false }),
-            ...(readOnly ? [] : [new HistoryExtension()]),
+            ...(readOnly ? [] : [new HistoryExtension({})]),
         ],
         content,
         stringHandler: 'markdown',
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/ChartDashboardsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/ChartDashboardsTab.tsx
index 89e017947519c..fc1ff4784dac7 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Entity/ChartDashboardsTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/ChartDashboardsTab.tsx
@@ -7,7 +7,7 @@ import { useEntityRegistry } from '../../../../useEntityRegistry';
 export const ChartDashboardsTab = () => {
     const entity = useBaseEntity() as any;
     const chart = entity && entity.chart;
-    const dashboards = chart?.dashboards?.relationships.map((relationship) => relationship.entity);
+    const dashboards = chart?.dashboards?.relationships?.map((relationship) => relationship.entity);
     const entityRegistry = useEntityRegistry();
     const totalDashboards = chart?.dashboards?.total || 0;
     const title = `Found in ${totalDashboards} ${
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardChartsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardChartsTab.tsx
index 7e5b1cdebd77b..1b099eb1beb54 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardChartsTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardChartsTab.tsx
@@ -7,7 +7,7 @@ import { useEntityRegistry } from '../../../../useEntityRegistry';
 export const DashboardChartsTab = () => {
     const entity = useBaseEntity() as any;
     const dashboard = entity && entity.dashboard;
-    const charts = dashboard?.charts?.relationships.map((relationship) => relationship.entity);
+    const charts = dashboard?.charts?.relationships?.map((relationship) => relationship.entity);
     const entityRegistry = useEntityRegistry();
     const totalCharts = dashboard?.charts?.total || 0;
     const title = `Contains ${totalCharts} ${
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx
index 62d612b0df4df..758c412e031db 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx
@@ -7,7 +7,7 @@ import { useEntityRegistry } from '../../../../useEntityRegistry';
 export const DashboardDatasetsTab = () => {
     const entity = useBaseEntity() as any;
     const dashboard = entity && entity.dashboard;
-    const datasets = dashboard?.datasets?.relationships.map((relationship) => relationship.entity);
+    const datasets = dashboard?.datasets?.relationships?.map((relationship) => relationship.entity);
     const entityRegistry = useEntityRegistry();
     const totalDatasets = dashboard?.datasets?.total || 0;
     const title = `Consumes ${totalDatasets} ${
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/DataFlowJobsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/DataFlowJobsTab.tsx
index 36e8c035c0d23..cccf8e1449d83 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Entity/DataFlowJobsTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/DataFlowJobsTab.tsx
@@ -30,7 +30,7 @@ export const DataFlowJobsTab = ({ properties = { urn: '' } }: Props) => {
     };
 
     const dataFlow = data && data?.dataFlow;
-    const dataJobs = dataFlow?.childJobs?.relationships.map((relationship) => relationship.entity);
+    const dataJobs = dataFlow?.childJobs?.relationships?.map((relationship) => relationship.entity);
     const entityRegistry = useEntityRegistry();
     const totalJobs = dataFlow?.childJobs?.total || 0;
     const pageSize = data?.dataFlow?.childJobs?.count || 0;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicPropertiesTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicPropertiesTab.tsx
index 81a9398afa418..aea879d72376f 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicPropertiesTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicPropertiesTab.tsx
@@ -25,7 +25,7 @@ export default function DynamicTabularTab({ payload: rawPayload }: Props) {
             width: 210,
             title: 'Name',
             dataIndex: 'key',
-            sorter: (a, b) => a?.key.localeCompare(b?.key || '') || 0,
+            sorter: (a, b) => a?.key?.localeCompare(b?.key || '') || 0,
             defaultSortOrder: 'ascend',
             render: (name: string) => <NameText>{name}</NameText>,
         },
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx
index fda8c9cda2d0d..663575f062093 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx
@@ -1,7 +1,7 @@
 import React, { useState } from 'react';
 import { message, Modal, Button, Form, Input, Typography, Select } from 'antd';
 import { useApolloClient } from '@apollo/client';
-import TextArea from 'antd/lib/input/TextArea';
+import styled from 'styled-components';
 import analytics, { EventType, EntityActionType } from '../../../../../analytics';
 import { useEntityData } from '../../../EntityContext';
 import { EntityType, IncidentSourceType, IncidentState, IncidentType } from '../../../../../../types.generated';
@@ -9,6 +9,12 @@ import { INCIDENT_DISPLAY_TYPES, PAGE_SIZE, addActiveIncidentToCache } from '../
 import { useRaiseIncidentMutation } from '../../../../../../graphql/mutations.generated';
 import handleGraphQLError from '../../../../../shared/handleGraphQLError';
 import { useUserContext } from '../../../../../context/useUserContext';
+import { Editor } from '../../Documentation/components/editor/Editor';
+import { ANTD_GRAY } from '../../../constants';
+
+const StyledEditor = styled(Editor)`
+    border: 1px solid ${ANTD_GRAY[4.5]};
+`;
 
 type AddIncidentProps = {
     open: boolean;
@@ -112,6 +118,7 @@ export const AddIncidentModal = ({ open, onClose, refetch }: AddIncidentProps) =
                 open={open}
                 destroyOnClose
                 onCancel={handleClose}
+                width={600}
                 footer={[
                     <Button type="text" onClick={handleClose}>
                         Cancel
@@ -174,7 +181,17 @@ export const AddIncidentModal = ({ open, onClose, refetch }: AddIncidentProps) =
                             },
                         ]}
                     >
-                        <TextArea placeholder="Provide some additional details" />
+                        <StyledEditor
+                            doNotFocus
+                            className="add-incident-description"
+                            onKeyDown={(e) => {
+                                // Preventing the modal from closing when the Enter key is pressed
+                                if (e.key === 'Enter') {
+                                    e.preventDefault();
+                                    e.stopPropagation();
+                                }
+                            }}
+                        />
                     </Form.Item>
                 </Form>
             </Modal>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx
index dc123398cd2da..5c413b1e8b3e8 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx
@@ -14,6 +14,7 @@ import { useUpdateIncidentStatusMutation } from '../../../../../../graphql/mutat
 import { ResolveIncidentModal } from './ResolveIncidentModal';
 import handleGraphQLError from '../../../../../shared/handleGraphQLError';
 import { MenuItemStyle } from '../../../../view/menu/item/styledComponent';
+import MarkdownViewer from '../../../components/legacy/MarkdownViewer';
 
 type Props = {
     incident: any;
@@ -227,12 +228,12 @@ export default function IncidentListItem({ incident, refetch }: Props) {
                             </TitleContainer>
                             <DescriptionContainer>
                                 <IncidentDescriptionLabel>Description</IncidentDescriptionLabel>
-                                <IncidentDescriptionText>{incident?.description}</IncidentDescriptionText>
+                                <MarkdownViewer source={incident?.description} />
                                 {incident.status.state === IncidentState.Resolved ? (
                                     <>
                                         <IncidentDescriptionLabel>Resolution Note</IncidentDescriptionLabel>
                                         <IncidentDescriptionText>
-                                            {incident?.status.message || 'No additional details'}
+                                            {incident?.status?.message || 'No additional details'}
                                         </IncidentDescriptionText>
                                     </>
                                 ) : null}
@@ -263,15 +264,15 @@ export default function IncidentListItem({ incident, refetch }: Props) {
                                 placement="left"
                                 title={<Typography.Text strong>Note</Typography.Text>}
                                 content={
-                                    incident?.status.message === null ? (
+                                    incident?.status?.message === null ? (
                                         <Typography.Text type="secondary">No additional details</Typography.Text>
                                     ) : (
-                                        <Typography.Text type="secondary">{incident?.status.message}</Typography.Text>
+                                        <Typography.Text type="secondary">{incident?.status?.message}</Typography.Text>
                                     )
                                 }
                             >
                                 <IncidentResolvedText>
-                                    {incident?.status.lastUpdated && (
+                                    {incident?.status?.lastUpdated && (
                                         <Tooltip showArrow={false} title={toLocalDateTimeString(lastModifiedDate)}>
                                             Resolved {toRelativeTimeString(lastModifiedDate)} by{' '}
                                         </Tooltip>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Lineage/ColumnLineageSelect.tsx b/datahub-web-react/src/app/entity/shared/tabs/Lineage/ColumnLineageSelect.tsx
index ebde070e38781..3b626181f80b1 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Lineage/ColumnLineageSelect.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Lineage/ColumnLineageSelect.tsx
@@ -70,7 +70,7 @@ export default function ColumnsLineageSelect({
                     allowClear
                     placeholder="Select column"
                 >
-                    {entityWithSchema?.schemaMetadata?.fields.map((field) => {
+                    {entityWithSchema?.schemaMetadata?.fields?.map((field) => {
                         const fieldPath = downgradeV2FieldPath(field.fieldPath);
                         return (
                             <Select.Option value={field.fieldPath}>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Lineage/generateUseDownloadScrollAcrossLineageSearchResultsHook.ts b/datahub-web-react/src/app/entity/shared/tabs/Lineage/generateUseDownloadScrollAcrossLineageSearchResultsHook.ts
index ab1957f681b83..156471a6dd7f0 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Lineage/generateUseDownloadScrollAcrossLineageSearchResultsHook.ts
+++ b/datahub-web-react/src/app/entity/shared/tabs/Lineage/generateUseDownloadScrollAcrossLineageSearchResultsHook.ts
@@ -30,11 +30,11 @@ export default function generateUseDownloadSearchAcrossLineageSearchResultsHook(
                 input: {
                     urn,
                     direction,
-                    types: params.variables?.input.types,
-                    query: params.variables?.input.query,
-                    scrollId: params.variables?.input.scrollId,
-                    count: params.variables?.input.count,
-                    orFilters: params.variables?.input.orFilters,
+                    types: params.variables?.input?.types,
+                    query: params.variables?.input?.query,
+                    scrollId: params.variables?.input?.scrollId,
+                    count: params.variables?.input?.count,
+                    orFilters: params.variables?.input?.orFilters,
                     startTimeMillis: startTimeMillis || undefined,
                     endTimeMillis: endTimeMillis || undefined,
                 },
@@ -44,8 +44,8 @@ export default function generateUseDownloadSearchAcrossLineageSearchResultsHook(
         return {
             searchResults: (data?.scrollAcrossLineage && {
                 ...data?.scrollAcrossLineage,
-                nextScrollId: data?.scrollAcrossLineage.nextScrollId,
-                searchResults: data?.scrollAcrossLineage.searchResults,
+                nextScrollId: data?.scrollAcrossLineage?.nextScrollId,
+                searchResults: data?.scrollAcrossLineage?.searchResults,
             }) as DownloadSearchResults,
             loading,
             error,
diff --git a/datahub-web-react/src/app/entity/shared/tabs/ML/MlFeatureFeatureTableTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/ML/MlFeatureFeatureTableTab.tsx
index 2930232b4c620..bf2e42e8eaf9d 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/ML/MlFeatureFeatureTableTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/ML/MlFeatureFeatureTableTab.tsx
@@ -10,7 +10,7 @@ export const FeatureTableTab = () => {
     const entityRegistry = useEntityRegistry();
 
     const feature = entity && entity.mlFeature;
-    const featureTables = feature?.featureTables?.relationships.map((relationship) => relationship.entity);
+    const featureTables = feature?.featureTables?.relationships?.map((relationship) => relationship.entity);
 
     const title = `Part of ${entityRegistry.getEntityName(EntityType.MlfeatureTable)}`;
     return <EntityList title={title} type={EntityType.MlfeatureTable} entities={featureTables || []} />;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/ML/MlPrimaryKeyFeatureTableTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/ML/MlPrimaryKeyFeatureTableTab.tsx
index 185e60c27fc80..ecad39fd45e57 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/ML/MlPrimaryKeyFeatureTableTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/ML/MlPrimaryKeyFeatureTableTab.tsx
@@ -10,7 +10,7 @@ export const FeatureTableTab = () => {
     const entityRegistry = useEntityRegistry();
 
     const feature = entity && entity.mlPrimaryKey;
-    const featureTables = feature?.featureTables?.relationships.map((relationship) => relationship.entity);
+    const featureTables = feature?.featureTables?.relationships?.map((relationship) => relationship.entity);
 
     const title = `Part of ${entityRegistry.getEntityName(EntityType.MlfeatureTable)}`;
     return <EntityList title={title} type={EntityType.MlfeatureTable} entities={featureTables || []} />;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/AddPropertyButton.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/AddPropertyButton.tsx
new file mode 100644
index 0000000000000..cac3e268c1df5
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/AddPropertyButton.tsx
@@ -0,0 +1,228 @@
+import { LoadingOutlined } from '@ant-design/icons';
+import { colors, Icon, Input as InputComponent, Text } from '@src/alchemy-components';
+import { useUserContext } from '@src/app/context/useUserContext';
+import { getEntityTypesPropertyFilter, getNotHiddenPropertyFilter } from '@src/app/govern/structuredProperties/utils';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { PageRoutes } from '@src/conf/Global';
+import { useGetSearchResultsForMultipleQuery } from '@src/graphql/search.generated';
+import { Dropdown } from 'antd';
+import { Tooltip } from '@components';
+import { EntityType, Maybe, StructuredProperties, StructuredPropertyEntity } from '@src/types.generated';
+import React, { useMemo, useState } from 'react';
+import { Link } from 'react-router-dom';
+import styled from 'styled-components';
+import { useEntityData } from '../../EntityContext';
+import EditStructuredPropertyModal from './Edit/EditStructuredPropertyModal';
+
+const AddButton = styled.div<{ isV1Drawer?: boolean }>`
+    border-radius: 200px;
+    background-color: #5280e2;
+    width: ${(props) => (props.isV1Drawer ? '24px' : '32px')};
+    height: ${(props) => (props.isV1Drawer ? '24px' : '32px')};
+    display: flex;
+    align-items: center;
+    justify-content: center;
+
+    :hover {
+        cursor: pointer;
+    }
+`;
+
+const DropdownContainer = styled.div`
+    border-radius: 12px;
+    box-shadow: 0px 0px 14px 0px rgba(0, 0, 0, 0.15);
+    background-color: ${colors.white};
+    padding-bottom: 8px;
+    width: 300px;
+`;
+
+const SearchContainer = styled.div`
+    padding: 8px;
+`;
+
+const OptionsContainer = styled.div`
+    max-height: 200px;
+    overflow-y: auto;
+    font-size: 14px;
+`;
+
+const Option = styled.div``;
+
+const LoadingContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    gap: 8px;
+    align-items: center;
+    justify-content: center;
+    height: 100px;
+`;
+
+const EmptyContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    gap: 8px;
+    align-items: center;
+    justify-content: center;
+    min-height: 50px;
+    padding: 16px;
+    text-align: center;
+`;
+
+interface Props {
+    fieldUrn?: string;
+    refetch?: () => void;
+    fieldProperties?: Maybe<StructuredProperties>;
+    isV1Drawer?: boolean;
+}
+
+const AddPropertyButton = ({ fieldUrn, refetch, fieldProperties, isV1Drawer }: Props) => {
+    const [searchQuery, setSearchQuery] = useState('');
+    const { entityData, entityType } = useEntityData();
+    const me = useUserContext();
+    const entityRegistry = useEntityRegistry();
+    const [isEditModalVisible, setIsEditModalVisible] = useState(false);
+
+    const inputs = {
+        types: [EntityType.StructuredProperty],
+        query: '',
+        start: 0,
+        count: 100,
+        searchFlags: { skipCache: true },
+        orFilters: [
+            {
+                and: [
+                    getEntityTypesPropertyFilter(entityRegistry, !!fieldUrn, entityType),
+                    getNotHiddenPropertyFilter(),
+                ],
+            },
+        ],
+    };
+
+    // Execute search
+    const { data, loading } = useGetSearchResultsForMultipleQuery({
+        variables: {
+            input: inputs,
+        },
+        fetchPolicy: 'cache-first',
+    });
+
+    const [selectedProperty, setSelectedProperty] = useState<StructuredPropertyEntity | undefined>(
+        data?.searchAcrossEntities?.searchResults?.[0]?.entity as StructuredPropertyEntity | undefined,
+    );
+
+    const handleOptionClick = (property: StructuredPropertyEntity) => {
+        setSelectedProperty(property);
+        setIsEditModalVisible(true);
+    };
+
+    const entityPropertiesUrns = entityData?.structuredProperties?.properties?.map(
+        (prop) => prop.structuredProperty.urn,
+    );
+    const fieldPropertiesUrns = fieldProperties?.properties?.map((prop) => prop.structuredProperty.urn);
+
+    // filter out the existing properties when displaying in the list of add button
+    const properties = useMemo(
+        () =>
+            data?.searchAcrossEntities?.searchResults
+                .filter((result) =>
+                    fieldUrn
+                        ? !fieldPropertiesUrns?.includes(result.entity.urn)
+                        : !entityPropertiesUrns?.includes(result.entity.urn),
+                )
+                .map((prop) => {
+                    const entity = prop.entity as StructuredPropertyEntity;
+                    return {
+                        label: (
+                            <Option key={entity.urn} onClick={() => handleOptionClick(entity)}>
+                                <Text weight="semiBold" color="gray">
+                                    {entity.definition?.displayName}
+                                </Text>
+                            </Option>
+                        ),
+                        key: entity.urn,
+                        name: entity.definition?.displayName || entity.urn,
+                    };
+                }),
+        [data, fieldUrn, fieldPropertiesUrns, entityPropertiesUrns],
+    );
+
+    const canEditProperties =
+        entityData?.parent?.privileges?.canEditProperties || entityData?.privileges?.canEditProperties;
+
+    if (!canEditProperties) return null;
+
+    // Filter items based on search query
+    const filteredItems = properties?.filter((prop) => prop.name?.toLowerCase().includes(searchQuery.toLowerCase()));
+
+    const noDataText =
+        properties?.length === 0 ? (
+            <>
+                It looks like there are no structured properties for this asset type.
+                {me.platformPrivileges?.manageStructuredProperties && (
+                    <span>
+                        {' '}
+                        <Link to={PageRoutes.STRUCTURED_PROPERTIES}>Manage custom properties</Link>
+                    </span>
+                )}
+            </>
+        ) : null;
+
+    return (
+        <>
+            <Dropdown
+                trigger={['click']}
+                menu={{ items: filteredItems }}
+                dropdownRender={(menuNode) => (
+                    <DropdownContainer>
+                        <SearchContainer>
+                            <InputComponent
+                                label=""
+                                placeholder="Search..."
+                                value={searchQuery}
+                                onChange={(e) => setSearchQuery(e.target.value)}
+                            />
+                        </SearchContainer>
+                        {loading ? (
+                            <LoadingContainer>
+                                <LoadingOutlined />
+                                <Text size="sm">Loading...</Text>
+                            </LoadingContainer>
+                        ) : (
+                            <>
+                                {filteredItems?.length === 0 && (
+                                    <EmptyContainer>
+                                        <Text color="gray" weight="medium">
+                                            No results found
+                                        </Text>
+                                        <Text size="sm" color="gray">
+                                            {noDataText}
+                                        </Text>
+                                    </EmptyContainer>
+                                )}
+                                <OptionsContainer>{menuNode}</OptionsContainer>
+                            </>
+                        )}
+                    </DropdownContainer>
+                )}
+            >
+                <Tooltip title="Add property" placement="left" showArrow={false}>
+                    <AddButton isV1Drawer={isV1Drawer} data-testid="add-structured-prop-button">
+                        <Icon icon="Add" size={isV1Drawer ? 'lg' : '2xl'} color="white" />
+                    </AddButton>
+                </Tooltip>
+            </Dropdown>
+            {selectedProperty && (
+                <EditStructuredPropertyModal
+                    isOpen={isEditModalVisible}
+                    closeModal={() => setIsEditModalVisible(false)}
+                    structuredProperty={selectedProperty}
+                    associatedUrn={fieldUrn} // pass in fieldUrn to use otherwise we will use mutation urn for siblings
+                    refetch={refetch}
+                    isAddMode
+                />
+            )}
+        </>
+    );
+};
+
+export default AddPropertyButton;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditColumn.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditColumn.tsx
index 6a0599c0cdb33..a5eafbae35399 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditColumn.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditColumn.tsx
@@ -1,27 +1,128 @@
-import { Button } from 'antd';
+import { colors, Icon, Text } from '@src/alchemy-components';
+import analytics, { EventType } from '@src/app/analytics';
+import { MenuItem } from '@src/app/govern/structuredProperties/styledComponents';
+import { ConfirmationModal } from '@src/app/sharedV2/modals/ConfirmationModal';
+import { showToastMessage, ToastType } from '@src/app/sharedV2/toastMessageUtils';
+import { useRemoveStructuredPropertiesMutation } from '@src/graphql/structuredProperties.generated';
+import { EntityType, StructuredPropertyEntity } from '@src/types.generated';
+import { Dropdown } from 'antd';
 import React, { useState } from 'react';
+import styled from 'styled-components';
+import { useEntityContext, useEntityData, useMutationUrn } from '../../../EntityContext';
 import EditStructuredPropertyModal from './EditStructuredPropertyModal';
-import { StructuredPropertyEntity } from '../../../../../../types.generated';
+
+export const MoreOptionsContainer = styled.div`
+    display: flex;
+    gap: 12px;
+    justify-content: end;
+
+    div {
+        background-color: ${colors.gray[1500]};
+        border-radius: 20px;
+        width: 24px;
+        height: 24px;
+        padding: 3px;
+        color: ${colors.gray[1800]};
+        :hover {
+            cursor: pointer;
+        }
+    }
+`;
 
 interface Props {
     structuredProperty?: StructuredPropertyEntity;
     associatedUrn?: string;
     values?: (string | number | null)[];
     refetch?: () => void;
+    isAddMode?: boolean;
 }
 
-export function EditColumn({ structuredProperty, associatedUrn, values, refetch }: Props) {
+export function EditColumn({ structuredProperty, associatedUrn, values, refetch, isAddMode }: Props) {
     const [isEditModalVisible, setIsEditModalVisible] = useState(false);
+    const { refetch: entityRefetch } = useEntityContext();
+    const { entityType } = useEntityData();
 
-    if (!structuredProperty || structuredProperty?.definition.immutable) {
+    const [removeStructuredProperty] = useRemoveStructuredPropertiesMutation();
+
+    const [showConfirmRemove, setShowConfirmRemove] = useState<boolean>(false);
+    const mutationUrn = useMutationUrn();
+
+    if (!structuredProperty || structuredProperty?.definition?.immutable) {
         return null;
     }
 
+    const handleRemoveProperty = () => {
+        showToastMessage(ToastType.LOADING, 'Removing structured property', 1);
+        removeStructuredProperty({
+            variables: {
+                input: {
+                    assetUrn: associatedUrn || mutationUrn,
+                    structuredPropertyUrns: [structuredProperty.urn],
+                },
+            },
+        })
+            .then(() => {
+                analytics.event({
+                    type: EventType.RemoveStructuredPropertyEvent,
+                    propertyUrn: structuredProperty.urn,
+                    propertyType: structuredProperty.definition.valueType.urn,
+                    assetUrn: associatedUrn || mutationUrn,
+                    assetType: associatedUrn?.includes('urn:li:schemaField') ? EntityType.SchemaField : entityType,
+                });
+                showToastMessage(ToastType.SUCCESS, 'Structured property removed successfully!', 3);
+                if (refetch) {
+                    refetch();
+                } else {
+                    entityRefetch();
+                }
+            })
+            .catch(() => {
+                showToastMessage(ToastType.ERROR, 'Failed to remove structured property', 3);
+            });
+
+        setShowConfirmRemove(false);
+    };
+
+    const handleRemoveClose = () => {
+        setShowConfirmRemove(false);
+    };
+
+    const items = [
+        {
+            key: '0',
+            label: (
+                <MenuItem
+                    onClick={() => {
+                        setIsEditModalVisible(true);
+                    }}
+                >
+                    {isAddMode ? 'Add' : 'Edit'}
+                </MenuItem>
+            ),
+        },
+    ];
+    if (values && values?.length > 0) {
+        items.push({
+            key: '1',
+            label: (
+                <MenuItem
+                    onClick={() => {
+                        setShowConfirmRemove(true);
+                    }}
+                >
+                    <Text color="red"> Remove </Text>
+                </MenuItem>
+            ),
+        });
+    }
+
     return (
         <>
-            <Button type="link" onClick={() => setIsEditModalVisible(true)}>
-                Edit
-            </Button>
+            <MoreOptionsContainer>
+                <Dropdown menu={{ items }} trigger={['click']}>
+                    <Icon icon="MoreVert" size="md" data-testid="structured-prop-entity-more-icon" />
+                </Dropdown>
+            </MoreOptionsContainer>
             <EditStructuredPropertyModal
                 isOpen={isEditModalVisible}
                 structuredProperty={structuredProperty}
@@ -29,6 +130,14 @@ export function EditColumn({ structuredProperty, associatedUrn, values, refetch
                 values={values}
                 closeModal={() => setIsEditModalVisible(false)}
                 refetch={refetch}
+                isAddMode={isAddMode}
+            />
+            <ConfirmationModal
+                isOpen={showConfirmRemove}
+                handleClose={handleRemoveClose}
+                handleConfirm={() => handleRemoveProperty()}
+                modalTitle="Confirm Remove Structured Property"
+                modalText={`Are you sure you want to remove ${structuredProperty.definition.displayName} from this asset?`}
             />
         </>
     );
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditStructuredPropertyModal.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditStructuredPropertyModal.tsx
index c8def8bef5e19..92691b1ad2239 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditStructuredPropertyModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditStructuredPropertyModal.tsx
@@ -1,12 +1,13 @@
+import analytics, { EventType } from '@src/app/analytics';
 import { Button, Modal, message } from 'antd';
 import React, { useEffect, useMemo } from 'react';
 import styled from 'styled-components';
-import StructuredPropertyInput from '../../../components/styled/StructuredProperty/StructuredPropertyInput';
-import { PropertyValueInput, StructuredPropertyEntity } from '../../../../../../types.generated';
 import { useUpsertStructuredPropertiesMutation } from '../../../../../../graphql/structuredProperties.generated';
-import { useEditStructuredProperty } from '../../../components/styled/StructuredProperty/useEditStructuredProperty';
-import { useEntityContext, useMutationUrn } from '../../../EntityContext';
+import { EntityType, PropertyValueInput, StructuredPropertyEntity } from '../../../../../../types.generated';
 import handleGraphQLError from '../../../../../shared/handleGraphQLError';
+import StructuredPropertyInput from '../../../components/styled/StructuredProperty/StructuredPropertyInput';
+import { useEditStructuredProperty } from '../../../components/styled/StructuredProperty/useEditStructuredProperty';
+import { useEntityContext, useEntityData, useMutationUrn } from '../../../EntityContext';
 
 const Description = styled.div`
     font-size: 14px;
@@ -21,6 +22,7 @@ interface Props {
     values?: (string | number | null)[];
     closeModal: () => void;
     refetch?: () => void;
+    isAddMode?: boolean;
 }
 
 export default function EditStructuredPropertyModal({
@@ -30,9 +32,11 @@ export default function EditStructuredPropertyModal({
     values,
     closeModal,
     refetch,
+    isAddMode,
 }: Props) {
     const { refetch: entityRefetch } = useEntityContext();
     const mutationUrn = useMutationUrn();
+    const { entityType } = useEntityData();
     const urn = associatedUrn || mutationUrn;
     const initialValues = useMemo(() => values || [], [values]);
     const { selectedValues, selectSingleValue, toggleSelectedValue, updateSelectedValues, setSelectedValues } =
@@ -44,7 +48,13 @@ export default function EditStructuredPropertyModal({
     }, [isOpen, initialValues, setSelectedValues]);
 
     function upsertProperties() {
-        message.loading('Updating...');
+        message.loading(isAddMode ? 'Adding...' : 'Updating...');
+        const propValues = selectedValues.map((value) => {
+            if (typeof value === 'string') {
+                return { stringValue: value as string };
+            }
+            return { numberValue: value as number };
+        }) as PropertyValueInput[];
         upsertStructuredProperties({
             variables: {
                 input: {
@@ -52,25 +62,30 @@ export default function EditStructuredPropertyModal({
                     structuredPropertyInputParams: [
                         {
                             structuredPropertyUrn: structuredProperty.urn,
-                            values: selectedValues.map((value) => {
-                                if (typeof value === 'string') {
-                                    return { stringValue: value as string };
-                                }
-                                return { numberValue: value as number };
-                            }) as PropertyValueInput[],
+                            values: propValues,
                         },
                     ],
                 },
             },
         })
             .then(() => {
+                analytics.event({
+                    type: isAddMode
+                        ? EventType.ApplyStructuredPropertyEvent
+                        : EventType.UpdateStructuredPropertyOnAssetEvent,
+                    propertyUrn: structuredProperty.urn,
+                    propertyType: structuredProperty.definition.valueType.urn,
+                    assetUrn: urn,
+                    assetType: associatedUrn?.includes('urn:li:schemaField') ? EntityType.SchemaField : entityType,
+                    values: propValues,
+                });
                 if (refetch) {
                     refetch();
                 } else {
                     entityRefetch();
                 }
                 message.destroy();
-                message.success('Successfully updated structured property!');
+                message.success(`Successfully ${isAddMode ? 'added' : 'updated'} structured property!`);
                 closeModal();
             })
             .catch((error) => {
@@ -84,7 +99,7 @@ export default function EditStructuredPropertyModal({
 
     return (
         <Modal
-            title={structuredProperty.definition.displayName}
+            title={`${isAddMode ? 'Add property' : 'Edit property'} ${structuredProperty?.definition?.displayName}`}
             onCancel={closeModal}
             open={isOpen}
             width={650}
@@ -93,14 +108,19 @@ export default function EditStructuredPropertyModal({
                     <Button onClick={closeModal} type="text">
                         Cancel
                     </Button>
-                    <Button type="primary" onClick={upsertProperties} disabled={!selectedValues.length}>
-                        Update
+                    <Button
+                        type="primary"
+                        onClick={upsertProperties}
+                        disabled={!selectedValues.length}
+                        data-testid="add-update-structured-prop-on-entity-button"
+                    >
+                        {isAddMode ? 'Add' : 'Update'}
                     </Button>
                 </>
             }
             destroyOnClose
         >
-            {structuredProperty?.definition.description && (
+            {structuredProperty?.definition?.description && (
                 <Description>{structuredProperty.definition.description}</Description>
             )}
             <StructuredPropertyInput
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/NameColumn.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/NameColumn.tsx
index 3b718c1ec30ed..069ba991b98c6 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/NameColumn.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/NameColumn.tsx
@@ -77,7 +77,7 @@ export default function NameColumn({ propertyRow, filterText }: Props) {
                     ) : (
                         <span />
                     )}
-                    {structuredProperty?.definition.allowedValues && (
+                    {structuredProperty?.definition?.allowedValues && (
                         <CardinalityLabel structuredProperty={structuredProperty} />
                     )}
                 </NameLabelWrapper>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/PropertiesTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/PropertiesTab.tsx
index eeff8fc2e2795..5fc209688c957 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/PropertiesTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/PropertiesTab.tsx
@@ -44,6 +44,7 @@ export const PropertiesTab = () => {
             render: (propertyRow: PropertyRow) => (
                 <EditColumn
                     structuredProperty={propertyRow.structuredProperty}
+                    associatedUrn={propertyRow.associatedUrn}
                     values={propertyRow.values?.map((v) => v.value) || []}
                 />
             ),
@@ -51,9 +52,12 @@ export const PropertiesTab = () => {
     }
 
     const { structuredPropertyRows, expandedRowsFromFilter } = useStructuredProperties(entityRegistry, filterText);
+    const filteredStructuredPropertyRows = structuredPropertyRows.filter(
+        (row) => !row.structuredProperty?.settings?.isHidden,
+    );
     const customProperties = getFilteredCustomProperties(filterText, entityData) || [];
     const customPropertyRows = mapCustomPropertiesToPropertyRows(customProperties);
-    const dataSource: PropertyRow[] = structuredPropertyRows.concat(customPropertyRows);
+    const dataSource: PropertyRow[] = filteredStructuredPropertyRows.concat(customPropertyRows);
 
     const [expandedRows, setExpandedRows] = useState<Set<string>>(new Set());
 
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/StructuredPropertyValue.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/StructuredPropertyValue.tsx
index b1a01f2b69fe1..2ed4ab79a41ee 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/StructuredPropertyValue.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/StructuredPropertyValue.tsx
@@ -1,24 +1,26 @@
 import Icon from '@ant-design/icons/lib/components/Icon';
-import React, { useState } from 'react';
+import React from 'react';
 import Highlight from 'react-highlighter';
-import { Button, Typography } from 'antd';
+import { Typography } from 'antd';
 import styled from 'styled-components';
 import { ValueColumnData } from './types';
 import { ANTD_GRAY } from '../../constants';
 import { useEntityRegistry } from '../../../../useEntityRegistry';
 import ExternalLink from '../../../../../images/link-out.svg?react';
-import MarkdownViewer, { MarkdownView } from '../../components/legacy/MarkdownViewer';
+import CompactMarkdownViewer from '../Documentation/components/CompactMarkdownViewer';
 import EntityIcon from '../../components/styled/EntityIcon';
 
 const ValueText = styled(Typography.Text)`
     font-family: 'Manrope';
     font-weight: 400;
-    font-size: 14px;
+    font-size: 12px;
     color: ${ANTD_GRAY[9]};
     display: block;
+    width: 100%;
+    margin-bottom: 2px;
 
-    ${MarkdownView} {
-        font-size: 14px;
+    .remirror-editor.ProseMirror {
+        font-size: 12px;
     }
 `;
 
@@ -28,38 +30,56 @@ const StyledIcon = styled(Icon)`
 
 const IconWrapper = styled.span`
     margin-right: 4px;
+    display: flex;
 `;
 
-const StyledButton = styled(Button)`
-    margin-top: 2px;
+const EntityWrapper = styled.div`
+    display: flex;
+    align-items: center;
+`;
+
+const EntityName = styled(Typography.Text)`
+    overflow: hidden;
+    white-space: nowrap;
+    text-overflow: ellipsis;
+`;
+
+const StyledHighlight = styled(Highlight)<{ truncateText?: boolean }>`
+    line-height: 1.5;
+    text-wrap: wrap;
+
+    ${(props) =>
+        props.truncateText &&
+        `
+        overflow: hidden;
+        text-overflow: ellipsis;
+        white-space: nowrap;
+        max-width: 100%;
+        display: block;
+    `}
 `;
 
 interface Props {
     value: ValueColumnData;
     isRichText?: boolean;
     filterText?: string;
+    truncateText?: boolean;
+    isFieldColumn?: boolean;
 }
 
-const MAX_CHARACTERS = 200;
-
-export default function StructuredPropertyValue({ value, isRichText, filterText }: Props) {
+export default function StructuredPropertyValue({ value, isRichText, filterText, truncateText, isFieldColumn }: Props) {
     const entityRegistry = useEntityRegistry();
-    const [showMore, setShowMore] = useState(false);
-
-    const toggleShowMore = () => {
-        setShowMore(!showMore);
-    };
-
-    const valueAsString = value?.value?.toString() ?? '';
 
     return (
         <ValueText>
             {value.entity ? (
-                <>
+                <EntityWrapper>
                     <IconWrapper>
-                        <EntityIcon entity={value.entity} />
+                        <EntityIcon entity={value.entity} size={12} />
                     </IconWrapper>
-                    {entityRegistry.getDisplayName(value.entity.type, value.entity)}
+                    <EntityName ellipsis={{ tooltip: true }}>
+                        {entityRegistry.getDisplayName(value.entity.type, value.entity)}
+                    </EntityName>
                     <Typography.Link
                         href={entityRegistry.getEntityUrl(value.entity.type, value.entity.urn)}
                         target="_blank"
@@ -67,20 +87,26 @@ export default function StructuredPropertyValue({ value, isRichText, filterText
                     >
                         <StyledIcon component={ExternalLink} />
                     </Typography.Link>
-                </>
+                </EntityWrapper>
             ) : (
                 <>
                     {isRichText ? (
-                        <MarkdownViewer source={value.value as string} />
+                        <CompactMarkdownViewer
+                            content={value.value?.toString() ?? ''}
+                            lineLimit={isFieldColumn ? 1 : undefined}
+                            hideShowMore={isFieldColumn}
+                            scrollableY={!isFieldColumn}
+                        />
                     ) : (
                         <>
-                            <Highlight search={filterText}>
-                                {showMore ? valueAsString : valueAsString?.substring(0, MAX_CHARACTERS)}
-                            </Highlight>
-                            {valueAsString?.length > MAX_CHARACTERS && (
-                                <StyledButton type="link" onClick={toggleShowMore}>
-                                    {showMore ? 'Show less' : 'Show more'}
-                                </StyledButton>
+                            {truncateText ? (
+                                <Typography.Text ellipsis={{ tooltip: true }}>
+                                    {value.value?.toString() || <div style={{ minHeight: 22 }} />}
+                                </Typography.Text>
+                            ) : (
+                                <StyledHighlight search={filterText} truncateText={truncateText}>
+                                    {value.value?.toString() || <div style={{ minHeight: 22 }} />}
+                                </StyledHighlight>
                             )}
                         </>
                     )}
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/TabHeader.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/TabHeader.tsx
index 9e0b4992d9c78..192b840b50040 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/TabHeader.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/TabHeader.tsx
@@ -1,8 +1,10 @@
 import { SearchOutlined } from '@ant-design/icons';
+import { Maybe, StructuredProperties } from '@src/types.generated';
 import { Input } from 'antd';
 import React from 'react';
 import styled from 'styled-components';
 import { ANTD_GRAY } from '../../constants';
+import AddPropertyButton from './AddPropertyButton';
 
 const StyledInput = styled(Input)`
     border-radius: 70px;
@@ -12,13 +14,18 @@ const StyledInput = styled(Input)`
 const TableHeader = styled.div`
     padding: 8px 16px;
     border-bottom: 1px solid ${ANTD_GRAY[4.5]};
+    display: flex;
+    justify-content: space-between;
 `;
 
 interface Props {
     setFilterText: (text: string) => void;
+    fieldUrn?: string;
+    fieldProperties?: Maybe<StructuredProperties>;
+    refetch?: () => void;
 }
 
-export default function TabHeader({ setFilterText }: Props) {
+export default function TabHeader({ setFilterText, fieldUrn, fieldProperties, refetch }: Props) {
     return (
         <TableHeader>
             <StyledInput
@@ -27,6 +34,7 @@ export default function TabHeader({ setFilterText }: Props) {
                 allowClear
                 prefix={<SearchOutlined />}
             />
+            <AddPropertyButton fieldUrn={fieldUrn} fieldProperties={fieldProperties} refetch={refetch} />
         </TableHeader>
     );
 }
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/ValuesColumn.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/ValuesColumn.tsx
index b050e06f96de8..3e21f1b5e2870 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/ValuesColumn.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/ValuesColumn.tsx
@@ -10,7 +10,7 @@ interface Props {
 
 export default function ValuesColumn({ propertyRow, filterText }: Props) {
     const { values } = propertyRow;
-    const isRichText = propertyRow.dataType?.info.type === StdDataType.RichText;
+    const isRichText = propertyRow.dataType?.info?.type === StdDataType.RichText;
 
     return (
         <>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/__tests__/useStructuredProperties.test.ts b/datahub-web-react/src/app/entity/shared/tabs/Properties/__tests__/useStructuredProperties.test.ts
new file mode 100644
index 0000000000000..ff7c6e51a04a0
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/__tests__/useStructuredProperties.test.ts
@@ -0,0 +1,87 @@
+import { identifyAndAddParentRows } from '../useStructuredProperties';
+
+describe('identifyAndAddParentRows', () => {
+    it('should not return parent rows when there are none', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'test1' },
+            { displayName: 'test2', qualifiedName: 'test2' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([]);
+    });
+
+    it('should not return parent rows when another row starts with the same letters but is a different token', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'testing.one' },
+            { displayName: 'test2', qualifiedName: 'testingAgain.two' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([]);
+    });
+
+    it('should return parent rows properly', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'testing.one' },
+            { displayName: 'test2', qualifiedName: 'testing.two' },
+            { displayName: 'test3', qualifiedName: 'testing.three' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([
+            { displayName: 'testing', qualifiedName: 'testing', childrenCount: 3 },
+        ]);
+    });
+
+    it('should return parent rows properly with multiple layers of nesting', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'testing.one.two.a.1' },
+            { displayName: 'test1', qualifiedName: 'testing.one.two.a.2' },
+            { displayName: 'test1', qualifiedName: 'testing.one.two.b' },
+            { displayName: 'test1', qualifiedName: 'testing.one.three' },
+            { displayName: 'test2', qualifiedName: 'testing.two.c.d' },
+            { displayName: 'test3', qualifiedName: 'testing.three' },
+            { displayName: 'test3', qualifiedName: 'testParent' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([
+            { displayName: 'testing', qualifiedName: 'testing', isParentRow: true, childrenCount: 6 },
+            { displayName: 'testing.one', qualifiedName: 'testing.one', isParentRow: true, childrenCount: 4 },
+            { displayName: 'testing.one.two', qualifiedName: 'testing.one.two', isParentRow: true, childrenCount: 3 },
+            {
+                displayName: 'testing.one.two.a',
+                qualifiedName: 'testing.one.two.a',
+                isParentRow: true,
+                childrenCount: 2,
+            },
+        ]);
+    });
+
+    it('should return parent rows properly with multiple layers of nesting regardless of order', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'testing.one.two.a.1' },
+            { displayName: 'test3', qualifiedName: 'testParent' },
+            { displayName: 'test1', qualifiedName: 'testing.one.three' },
+            { displayName: 'test2', qualifiedName: 'testing.two.c.d' },
+            { displayName: 'test1', qualifiedName: 'testing.one.two.b' },
+            { displayName: 'test3', qualifiedName: 'testing.three' },
+            { displayName: 'test1', qualifiedName: 'testing.one.two.a.2' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([
+            { displayName: 'testing', qualifiedName: 'testing', isParentRow: true, childrenCount: 6 },
+            { displayName: 'testing.one', qualifiedName: 'testing.one', isParentRow: true, childrenCount: 4 },
+            { displayName: 'testing.one.two', qualifiedName: 'testing.one.two', isParentRow: true, childrenCount: 3 },
+            {
+                displayName: 'testing.one.two.a',
+                qualifiedName: 'testing.one.two.a',
+                isParentRow: true,
+                childrenCount: 2,
+            },
+        ]);
+    });
+
+    it('should return parent rows properly with simpler layers of nesting', () => {
+        const propertyRows = [
+            { displayName: 'test2', qualifiedName: 'testing.two.c.d' },
+            { displayName: 'test3', qualifiedName: 'testing.three' },
+            { displayName: 'test3', qualifiedName: 'testParent' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([
+            { displayName: 'testing', qualifiedName: 'testing', isParentRow: true, childrenCount: 2 },
+        ]);
+    });
+});
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/types.ts b/datahub-web-react/src/app/entity/shared/tabs/Properties/types.ts
index b93ba886d5a64..4adaafc3d98b6 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/types.ts
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/types.ts
@@ -22,4 +22,5 @@ export interface PropertyRow {
     dataType?: DataTypeEntity;
     isParentRow?: boolean;
     structuredProperty?: StructuredPropertyEntity;
+    associatedUrn?: string;
 }
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx
index 86365b8232905..60d0aac30eb4c 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx
@@ -23,6 +23,25 @@ export function mapStructuredPropertyValues(structuredPropertiesEntry: Structure
         }));
 }
 
+export function mapStructuredPropertyToPropertyRow(structuredPropertiesEntry: StructuredPropertiesEntry) {
+    const { displayName, qualifiedName } = structuredPropertiesEntry.structuredProperty.definition;
+    return {
+        displayName: displayName || qualifiedName,
+        qualifiedName,
+        values: mapStructuredPropertyValues(structuredPropertiesEntry),
+        dataType: structuredPropertiesEntry.structuredProperty.definition.valueType,
+        structuredProperty: structuredPropertiesEntry.structuredProperty,
+        type:
+            structuredPropertiesEntry.values[0] && structuredPropertiesEntry.values[0].__typename
+                ? {
+                      type: typeNameToType[structuredPropertiesEntry.values[0].__typename].type,
+                      nativeDataType: typeNameToType[structuredPropertiesEntry.values[0].__typename].nativeDataType,
+                  }
+                : undefined,
+        associatedUrn: structuredPropertiesEntry.associatedUrn,
+    };
+}
+
 // map the properties map into a list of PropertyRow objects to render in a table
 function getStructuredPropertyRows(entityData?: GenericEntityProperties | null) {
     const structuredPropertyRows: PropertyRow[] = [];
@@ -45,6 +64,7 @@ function getStructuredPropertyRows(entityData?: GenericEntityProperties | null)
                                   typeNameToType[structuredPropertiesEntry.values[0].__typename].nativeDataType,
                           }
                         : undefined,
+                associatedUrn: structuredPropertiesEntry.associatedUrn,
             });
         });
 
@@ -102,10 +122,10 @@ export function identifyAndAddParentRows(rows?: Array<PropertyRow>): Array<Prope
         // that would tell us to nest. If the count is not equal, we should nest the child properties.
         for (let index = 0; index < substrings.length; index++) {
             const token = substrings[index];
-            const currentCount = qualifiedNames.filter((name) => name.startsWith(token)).length;
+            const currentCount = qualifiedNames.filter((name) => name.startsWith(`${token}.`)).length;
 
-            // If we're at the beginning of the path and there is no nesting, break
-            if (index === 0 && currentCount === 1) {
+            // If there's only one child, don't nest it
+            if (currentCount === 1) {
                 break;
             }
 
diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts
index 00e501740c2ad..ceba5b4bf30eb 100644
--- a/datahub-web-react/src/app/entity/shared/types.ts
+++ b/datahub-web-react/src/app/entity/shared/types.ts
@@ -122,6 +122,7 @@ export type GenericEntityProperties = {
     browsePathV2?: Maybe<BrowsePathV2>;
     inputOutput?: Maybe<DataJobInputOutput>;
     forms?: Maybe<Forms>;
+    parent?: Maybe<GenericEntityProperties>;
 };
 
 export type GenericEntityUpdate = {
diff --git a/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx b/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx
new file mode 100644
index 0000000000000..2bdfd550652fa
--- /dev/null
+++ b/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx
@@ -0,0 +1,86 @@
+import * as React from 'react';
+import styled from 'styled-components';
+import TableIcon from '@src/images/table-icon.svg?react';
+import { TYPE_ICON_CLASS_NAME } from '@src/app/shared/constants';
+import DefaultPreviewCard from '@src/app/preview/DefaultPreviewCard';
+import { EntityType, SearchResult, StructuredPropertyEntity as StructuredProperty } from '../../../types.generated';
+import { Entity, IconStyleType, PreviewType } from '../Entity';
+import { getDataForEntityType } from '../shared/containers/profile/utils';
+import { urlEncodeUrn } from '../shared/utils';
+
+const PreviewPropIcon = styled(TableIcon)`
+    font-size: 20px;
+`;
+
+/**
+ * Definition of the DataHub Structured Property entity.
+ */
+export class StructuredPropertyEntity implements Entity<StructuredProperty> {
+    type: EntityType = EntityType.StructuredProperty;
+
+    icon = (fontSize?: number, styleType?: IconStyleType, color?: string) => {
+        if (styleType === IconStyleType.TAB_VIEW) {
+            return <TableIcon className={TYPE_ICON_CLASS_NAME} style={{ fontSize, color }} />;
+        }
+
+        if (styleType === IconStyleType.HIGHLIGHT) {
+            return <TableIcon className={TYPE_ICON_CLASS_NAME} style={{ fontSize, color: color || '#B37FEB' }} />;
+        }
+
+        return (
+            <TableIcon
+                className={TYPE_ICON_CLASS_NAME}
+                style={{
+                    fontSize,
+                    color: color || '#BFBFBF',
+                }}
+            />
+        );
+    };
+
+    isSearchEnabled = () => false;
+
+    isBrowseEnabled = () => false;
+
+    isLineageEnabled = () => false;
+
+    getAutoCompleteFieldName = () => 'name';
+
+    getGraphName = () => 'structuredProperty';
+
+    getPathName: () => string = () => this.getGraphName();
+
+    getCollectionName: () => string = () => 'Structured Properties';
+
+    getEntityName: () => string = () => 'Structured Property';
+
+    renderProfile: (urn: string) => JSX.Element = (_urn) => <div />; // not used right now
+
+    renderPreview = (previewType: PreviewType, data: StructuredProperty) => (
+        <DefaultPreviewCard
+            description={data.definition?.description || ''}
+            name={this.displayName(data)}
+            urn={data.urn}
+            url={`/${this.getPathName()}/${urlEncodeUrn(data.urn)}`}
+            logoComponent={<PreviewPropIcon />}
+            typeIcon={this.icon(14, IconStyleType.ACCENT)}
+            previewType={previewType}
+        />
+    );
+
+    renderSearch = (result: SearchResult) => {
+        return this.renderPreview(PreviewType.SEARCH, result.entity as StructuredProperty);
+    };
+
+    displayName = (data: StructuredProperty) => {
+        return data.definition?.displayName || data.definition?.qualifiedName || data.urn;
+    };
+
+    getGenericEntityProperties = (entity: StructuredProperty) => {
+        return getDataForEntityType({ data: entity, entityType: this.type, getOverrideProperties: (data) => data });
+    };
+
+    supportedCapabilities = () => {
+        return new Set([]);
+    };
+}
diff --git a/datahub-web-react/src/app/entity/tag/Tag.tsx b/datahub-web-react/src/app/entity/tag/Tag.tsx
index 6f0839e5f812b..d3c5b07966099 100644
--- a/datahub-web-react/src/app/entity/tag/Tag.tsx
+++ b/datahub-web-react/src/app/entity/tag/Tag.tsx
@@ -46,6 +46,8 @@ export class TagEntity implements Entity<Tag> {
 
     getAutoCompleteFieldName = () => 'name';
 
+    getGraphName = () => 'tag';
+
     getPathName: () => string = () => 'tag';
 
     getCollectionName: () => string = () => 'Tags';
diff --git a/datahub-web-react/src/app/entity/user/User.tsx b/datahub-web-react/src/app/entity/user/User.tsx
index ec1c5fbdc8698..058349f83eaec 100644
--- a/datahub-web-react/src/app/entity/user/User.tsx
+++ b/datahub-web-react/src/app/entity/user/User.tsx
@@ -39,6 +39,8 @@ export class UserEntity implements Entity<CorpUser> {
 
     getAutoCompleteFieldName = () => 'username';
 
+    getGraphName: () => string = () => 'corpuser';
+
     getPathName: () => string = () => 'user';
 
     getEntityName = () => 'Person';
diff --git a/datahub-web-react/src/app/entity/user/UserProfile.tsx b/datahub-web-react/src/app/entity/user/UserProfile.tsx
index 2d6efb95b9d2a..57d02031f4f95 100644
--- a/datahub-web-react/src/app/entity/user/UserProfile.tsx
+++ b/datahub-web-react/src/app/entity/user/UserProfile.tsx
@@ -65,9 +65,9 @@ export default function UserProfile() {
     const castedCorpUser = data?.corpUser as any;
 
     const userGroups: Array<EntityRelationship> =
-        castedCorpUser?.groups?.relationships.map((relationship) => relationship as EntityRelationship) || [];
+        castedCorpUser?.groups?.relationships?.map((relationship) => relationship as EntityRelationship) || [];
     const userRoles: Array<EntityRelationship> =
-        castedCorpUser?.roles?.relationships.map((relationship) => relationship as EntityRelationship) || [];
+        castedCorpUser?.roles?.relationships?.map((relationship) => relationship as EntityRelationship) || [];
 
     // Routed Tabs Constants
     const getTabs = () => {
diff --git a/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx b/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx
index 8a0b84ce0fa7f..9d8ba3e7a7392 100644
--- a/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx
+++ b/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx
@@ -32,7 +32,9 @@ export const ViewBuilder = ({ mode, urn, initialState, onSubmit, onCancel }: Pro
     const [createViewMutation] = useCreateViewMutation();
 
     const emitTrackingEvent = (viewUrn: string, state: ViewBuilderState, isCreate: boolean) => {
-        const filterFields = Array.from(new Set(state.definition?.filter?.filters.map((filter) => filter.field) ?? []));
+        const filterFields = Array.from(
+            new Set(state.definition?.filter?.filters?.map((filter) => filter.field) ?? []),
+        );
         const entityTypes = Array.from(new Set(state.definition?.entityTypes ?? []));
 
         analytics.event({
diff --git a/datahub-web-react/src/app/entity/view/utils.ts b/datahub-web-react/src/app/entity/view/utils.ts
index 68b64714c7ce9..0952cacb5ac45 100644
--- a/datahub-web-react/src/app/entity/view/utils.ts
+++ b/datahub-web-react/src/app/entity/view/utils.ts
@@ -28,7 +28,7 @@ export const convertStateToUpdateInput = (state: ViewBuilderState) => {
             entityTypes: state?.definition?.entityTypes,
             filter: {
                 operator: state?.definition?.filter?.operator,
-                filters: state?.definition?.filter?.filters.map((filter) => ({
+                filters: state?.definition?.filter?.filters?.map((filter) => ({
                     field: filter.field,
                     condition: filter.condition,
                     values: filter.values,
diff --git a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx
index 4e424b776a8ce..78944f19caa84 100644
--- a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx
+++ b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx
@@ -70,10 +70,10 @@ function BusinessGlossaryPage() {
         setEntityData(null);
     }, [setEntityData]);
 
-    const terms = termsData?.getRootGlossaryTerms?.terms.sort((termA, termB) =>
+    const terms = termsData?.getRootGlossaryTerms?.terms?.sort((termA, termB) =>
         sortGlossaryTerms(entityRegistry, termA, termB),
     );
-    const nodes = nodesData?.getRootGlossaryNodes?.nodes.sort((nodeA, nodeB) =>
+    const nodes = nodesData?.getRootGlossaryNodes?.nodes?.sort((nodeA, nodeB) =>
         sortGlossaryNodes(entityRegistry, nodeA, nodeB),
     );
 
diff --git a/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx b/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx
index af3c7454425de..a01c6712cc71f 100644
--- a/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx
+++ b/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx
@@ -80,7 +80,7 @@ function NodeItem(props: Props) {
     });
 
     useEffect(() => {
-        if (openToEntity && entityData && entityData.parentNodes?.nodes.some((parent) => parent.urn === node.urn)) {
+        if (openToEntity && entityData && entityData.parentNodes?.nodes?.some((parent) => parent.urn === node.urn)) {
             setAreChildrenVisible(true);
         }
     }, [entityData, node.urn, openToEntity]);
diff --git a/datahub-web-react/src/app/glossary/__tests__/utils.test.ts b/datahub-web-react/src/app/glossary/__tests__/utils.test.ts
index b3deb552cc721..91f50f6dfd983 100644
--- a/datahub-web-react/src/app/glossary/__tests__/utils.test.ts
+++ b/datahub-web-react/src/app/glossary/__tests__/utils.test.ts
@@ -13,7 +13,7 @@ const glossaryTermWithParent = {
 describe('glossary utils tests', () => {
     it('should get the direct parent node urn in getParentNodeToUpdate for glossary nodes', () => {
         const parentNode = getParentNodeToUpdate(glossaryNode3 as any, EntityType.GlossaryNode);
-        expect(parentNode).toBe(glossaryNode3.parentNodes?.nodes[0].urn);
+        expect(parentNode).toBe(glossaryNode3.parentNodes?.nodes[0]?.urn);
     });
 
     it('should get the direct parent node urn in getParentNodeToUpdate for glossary terms', () => {
diff --git a/datahub-web-react/src/app/glossary/utils.ts b/datahub-web-react/src/app/glossary/utils.ts
index 4cfbb06b8a4f3..dc08d3ae3eab3 100644
--- a/datahub-web-react/src/app/glossary/utils.ts
+++ b/datahub-web-react/src/app/glossary/utils.ts
@@ -12,8 +12,8 @@ export function getGlossaryRootToUpdate(entityType: EntityType) {
 // Get the urns or special constants for root nodes or terms (above) that need to be refreshed in the Glossary
 // sidebar when making updates (edit name, create term/term group, delete term/term group, move entity)
 export function getParentNodeToUpdate(entityData: GenericEntityProperties | null, entityType: EntityType) {
-    return entityData?.parentNodes?.nodes.length
-        ? entityData?.parentNodes?.nodes[0].urn
+    return entityData?.parentNodes?.nodes?.length
+        ? entityData?.parentNodes?.nodes[0]?.urn
         : getGlossaryRootToUpdate(entityType);
 }
 
diff --git a/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx b/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx
new file mode 100644
index 0000000000000..620143258ef5f
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx
@@ -0,0 +1,64 @@
+import { Icon, Input, Text, Tooltip } from '@components';
+import { Collapse, Form } from 'antd';
+import React from 'react';
+import { CollapseHeader, FlexContainer, InputLabel, StyledCollapse } from './styledComponents';
+
+interface Props {
+    isEditMode: boolean;
+}
+
+const AdvancedOptions = ({ isEditMode }: Props) => {
+    return (
+        <StyledCollapse
+            ghost
+            expandIcon={({ isActive }) => (
+                <Icon icon="ChevronRight" color="gray" size="4xl" rotate={isActive ? '90' : '0'} />
+            )}
+            expandIconPosition="end"
+            defaultActiveKey={[]}
+        >
+            <Collapse.Panel
+                key={1}
+                header={
+                    <CollapseHeader>
+                        <Text weight="bold" color="gray">
+                            Advanced Options
+                        </Text>
+                    </CollapseHeader>
+                }
+                forceRender
+            >
+                <InputLabel>
+                    <FlexContainer>
+                        Qualified Name
+                        <Tooltip
+                            title="Optionally provide a dot-separated fully qualified name for this property. This name serves as an ID, and must be unique across properties"
+                            showArrow={false}
+                        >
+                            <Icon icon="Info" color="violet" size="lg" />
+                        </Tooltip>
+                    </FlexContainer>
+                </InputLabel>
+                <Tooltip
+                    title={isEditMode && 'Once a property is created, Qualified Name cannot be changed'}
+                    showArrow={false}
+                >
+                    <Form.Item
+                        name="qualifiedName"
+                        rules={[
+                            {
+                                pattern: /^[^\s]*$/,
+                                whitespace: true,
+                                message: 'Qualified name cannot contain spaces',
+                            },
+                        ]}
+                    >
+                        <Input label="" placeholder="Optional - Qualified Name" isDisabled={isEditMode} />
+                    </Form.Item>
+                </Tooltip>
+            </Collapse.Panel>
+        </StyledCollapse>
+    );
+};
+
+export default AdvancedOptions;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx b/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx
new file mode 100644
index 0000000000000..16c07e8257cd9
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx
@@ -0,0 +1,143 @@
+import { Button, Icon, Input, Text, TextArea } from '@src/alchemy-components';
+import { AllowedValue } from '@src/types.generated';
+import { Form, FormInstance } from 'antd';
+import { Tooltip } from '@components';
+import React, { useEffect, useRef } from 'react';
+import {
+    AddButtonContainer,
+    DeleteIconContainer,
+    FieldGroupContainer,
+    FormContainer,
+    InputLabel,
+    StyledDivider,
+    ValuesContainer,
+} from './styledComponents';
+import { PropValueField } from './utils';
+
+interface Props {
+    showAllowedValuesDrawer: boolean;
+    propType: PropValueField;
+    allowedValues: AllowedValue[] | undefined;
+    isEditMode: boolean;
+    noOfExistingValues: number;
+    form: FormInstance;
+}
+
+const AllowedValuesDrawer = ({
+    showAllowedValuesDrawer,
+    propType,
+    allowedValues,
+    isEditMode,
+    noOfExistingValues,
+    form,
+}: Props) => {
+    useEffect(() => {
+        form.setFieldsValue({ allowedValues: allowedValues || [{}] });
+    }, [form, showAllowedValuesDrawer, allowedValues]);
+
+    const containerRef = useRef<HTMLDivElement>(null);
+
+    // Scroll to the bottom to show the newly added fields
+    const scrollToBottom = () => {
+        if (containerRef.current) {
+            containerRef.current.scrollTop = containerRef.current.scrollHeight;
+        }
+    };
+
+    return (
+        <Form form={form}>
+            <Form.List name="allowedValues">
+                {(fields, { add, remove }) => (
+                    <FormContainer>
+                        {fields.length > 0 && (
+                            <ValuesContainer ref={containerRef} height={window.innerHeight}>
+                                {fields.map((field, index) => {
+                                    const isExisting = isEditMode && index < noOfExistingValues;
+
+                                    return (
+                                        <FieldGroupContainer key={field.name}>
+                                            <InputLabel>
+                                                Value
+                                                <Text color="red" weight="bold">
+                                                    *
+                                                </Text>
+                                            </InputLabel>
+                                            <Tooltip
+                                                title={isExisting && 'Editing existing allowed values is not permitted'}
+                                                showArrow={false}
+                                            >
+                                                <Form.Item
+                                                    {...field}
+                                                    name={[field.name, propType]}
+                                                    rules={[
+                                                        {
+                                                            required: true,
+                                                            message: 'Please enter the value!',
+                                                        },
+                                                    ]}
+                                                    key={`${field.name}.value`}
+                                                    validateTrigger={['onChange', 'onBlur']}
+                                                >
+                                                    <Input
+                                                        label=""
+                                                        placeholder="Provide a value"
+                                                        type={propType === 'numberValue' ? 'number' : 'text'}
+                                                        isDisabled={isExisting}
+                                                    />
+                                                </Form.Item>
+                                            </Tooltip>
+                                            <Form.Item
+                                                {...field}
+                                                name={[field.name, 'description']}
+                                                key={`${field.name}.desc`}
+                                            >
+                                                <TextArea
+                                                    label="Description"
+                                                    placeholder="Provide a description"
+                                                    isDisabled={isExisting}
+                                                />
+                                            </Form.Item>
+                                            {!isExisting && (
+                                                <DeleteIconContainer>
+                                                    <Tooltip
+                                                        title="Remove from the allowed values list"
+                                                        showArrow={false}
+                                                    >
+                                                        <Icon
+                                                            icon="Delete"
+                                                            onClick={() => remove(field.name)}
+                                                            color="gray"
+                                                            size="xl"
+                                                        />
+                                                    </Tooltip>
+                                                </DeleteIconContainer>
+                                            )}
+                                            {index < fields.length - 1 && <StyledDivider />}
+                                        </FieldGroupContainer>
+                                    );
+                                })}
+                            </ValuesContainer>
+                        )}
+
+                        <AddButtonContainer>
+                            <Tooltip title="Add a new value to the allowed list" showArrow={false}>
+                                <Button
+                                    onClick={() => {
+                                        add();
+                                        setTimeout(() => scrollToBottom(), 0);
+                                    }}
+                                    color="violet"
+                                    type="button"
+                                >
+                                    Add
+                                </Button>
+                            </Tooltip>
+                        </AddButtonContainer>
+                    </FormContainer>
+                )}
+            </Form.List>
+        </Form>
+    );
+};
+
+export default AllowedValuesDrawer;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesField.tsx b/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesField.tsx
new file mode 100644
index 0000000000000..9a71d309c101f
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesField.tsx
@@ -0,0 +1,77 @@
+import { Icon, Text, Tooltip } from '@components';
+import { AllowedValue } from '@src/types.generated';
+import React from 'react';
+import {
+    FieldLabel,
+    FlexContainer,
+    ItemsContainer,
+    RowContainer,
+    StyledIcon,
+    ValueListContainer,
+    ValuesList,
+    ValueType,
+    VerticalDivider,
+} from './styledComponents';
+import { isStringOrNumberTypeSelected, PropValueField } from './utils';
+
+interface Props {
+    selectedValueType: string;
+    allowedValues: AllowedValue[] | undefined;
+    valueField: PropValueField;
+    setShowAllowedValuesDrawer: React.Dispatch<React.SetStateAction<boolean>>;
+}
+
+const AllowedValuesField = ({ selectedValueType, allowedValues, valueField, setShowAllowedValuesDrawer }: Props) => {
+    return (
+        <>
+            {isStringOrNumberTypeSelected(selectedValueType) && (
+                <RowContainer>
+                    <FieldLabel>
+                        <FlexContainer>
+                            Allowed Values
+                            <Tooltip
+                                title="Define the set of valid values for this property. If none are provided, any value will be allowed"
+                                showArrow={false}
+                            >
+                                <Icon icon="Info" color="violet" size="lg" />
+                            </Tooltip>
+                        </FlexContainer>
+                    </FieldLabel>
+
+                    {allowedValues && allowedValues.length > 0 ? (
+                        <ItemsContainer>
+                            <ValuesList>
+                                {allowedValues.map((val, index) => {
+                                    return (
+                                        <>
+                                            <Text>{val[valueField]}</Text>
+                                            {index < allowedValues.length - 1 && <VerticalDivider type="vertical" />}
+                                        </>
+                                    );
+                                })}
+                            </ValuesList>
+                            <Tooltip title="Update allowed values" showArrow={false}>
+                                <StyledIcon
+                                    icon="ChevronRight"
+                                    color="gray"
+                                    onClick={() => setShowAllowedValuesDrawer(true)}
+                                />
+                            </Tooltip>
+                        </ItemsContainer>
+                    ) : (
+                        <ValueListContainer>
+                            Any
+                            <ValueType>{valueField === 'stringValue' ? 'text' : 'number'} </ValueType>
+                            value will be allowed
+                            <Tooltip title="Update allowed values" showArrow={false}>
+                                <Icon icon="Add" color="gray" onClick={() => setShowAllowedValuesDrawer(true)} />
+                            </Tooltip>
+                        </ValueListContainer>
+                    )}
+                </RowContainer>
+            )}
+        </>
+    );
+};
+
+export default AllowedValuesField;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/DisplayPreferences.tsx b/datahub-web-react/src/app/govern/structuredProperties/DisplayPreferences.tsx
new file mode 100644
index 0000000000000..95823de0f27c4
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/DisplayPreferences.tsx
@@ -0,0 +1,166 @@
+import { Icon, Pill, Switch, Text } from '@src/alchemy-components';
+import { ConfirmationModal } from '@src/app/sharedV2/modals/ConfirmationModal';
+import { AllowedValue, StructuredPropertyEntity } from '@src/types.generated';
+import { Collapse } from 'antd';
+import React, { useState } from 'react';
+import { useUpdateStructuredPropertyMutation } from '@src/graphql/structuredProperties.generated';
+import { CollapseHeader, StyledCollapse, StyledFormItem, TogglesContainer } from './styledComponents';
+import { getDisplayName, canBeAssetBadge, StructuredProp } from './utils';
+
+const SCHEMA_FIELD_URN = 'urn:li:entityType:datahub.schemaField';
+
+interface Props {
+    formValues: StructuredProp | undefined;
+    handleDisplaySettingChange: (settingField: string, value: boolean) => void;
+    selectedValueType: string;
+    refetchProperties: () => void;
+    allowedValues?: AllowedValue[];
+    badgeProperty?: StructuredPropertyEntity;
+}
+
+const DisplayPreferences = ({
+    formValues,
+    handleDisplaySettingChange,
+    selectedValueType,
+    refetchProperties,
+    allowedValues,
+    badgeProperty,
+}: Props) => {
+    const [updateProperty] = useUpdateStructuredPropertyMutation();
+    const [showReplaceBadge, setShowReplaceBadge] = useState<boolean>(false);
+
+    const handleReplaceClose = () => {
+        setShowReplaceBadge(false);
+    };
+
+    function updateBadgePropertyToOff() {
+        if (badgeProperty) {
+            updateProperty({
+                variables: { input: { urn: badgeProperty.urn, settings: { showAsAssetBadge: false } } },
+            }).then(() => refetchProperties());
+        }
+    }
+
+    const hasAssetBadgeEnabled = formValues?.settings?.showAsAssetBadge;
+    const showInColumnsTable = formValues?.settings?.showInColumnsTable;
+    const hasColumnEntityType = formValues?.entityTypes?.includes(SCHEMA_FIELD_URN);
+
+    return (
+        <>
+            <StyledCollapse
+                ghost
+                expandIcon={({ isActive }) => (
+                    <Icon icon="ChevronRight" color="gray" size="4xl" rotate={isActive ? '90' : '0'} />
+                )}
+                expandIconPosition="end"
+                defaultActiveKey={[1]}
+            >
+                <Collapse.Panel
+                    key={1}
+                    header={
+                        <CollapseHeader>
+                            <Text weight="bold" color="gray">
+                                Display Preferences
+                            </Text>
+                        </CollapseHeader>
+                    }
+                    forceRender
+                >
+                    <TogglesContainer>
+                        <StyledFormItem name={['settings', 'isHidden']}>
+                            <Switch
+                                label="Hide Property"
+                                size="sm"
+                                checked={formValues?.settings?.isHidden}
+                                onChange={(e) => handleDisplaySettingChange('isHidden', e.target.checked)}
+                                labelHoverText="If enabled, this property will be hidden everywhere"
+                                data-testid="structured-props-hide-switch"
+                            />
+                        </StyledFormItem>
+                        <StyledFormItem name={['settings', 'showInSearchFilters']}>
+                            <Switch
+                                label="Show in Search Filters"
+                                size="sm"
+                                checked={formValues?.settings?.showInSearchFilters}
+                                onChange={(e) => handleDisplaySettingChange('showInSearchFilters', e.target.checked)}
+                                isDisabled={formValues?.settings?.isHidden}
+                                labelHoverText="If enabled, this property will appear in search filters"
+                            />
+                        </StyledFormItem>
+                        <StyledFormItem name={['settings', 'showInAssetSummary']}>
+                            <Switch
+                                label="Show in Asset Sidebar"
+                                size="sm"
+                                checked={formValues?.settings?.showInAssetSummary}
+                                onChange={(e) => handleDisplaySettingChange('showInAssetSummary', e.target.checked)}
+                                isDisabled={formValues?.settings?.isHidden}
+                                labelHoverText="If enabled, this property will appear in asset sidebar"
+                            />
+                        </StyledFormItem>
+                        <StyledFormItem name={['settings', 'showAsAssetBadge']}>
+                            <Switch
+                                label="Show as Asset Badge"
+                                size="sm"
+                                checked={formValues?.settings?.showAsAssetBadge === true}
+                                onChange={(e) => {
+                                    if (badgeProperty && e.target.checked) setShowReplaceBadge(true);
+                                    else handleDisplaySettingChange('showAsAssetBadge', e.target.checked);
+                                }}
+                                isDisabled={
+                                    !hasAssetBadgeEnabled &&
+                                    (formValues?.settings?.isHidden ||
+                                        !canBeAssetBadge(selectedValueType, allowedValues))
+                                }
+                                labelHoverText="If enabled, this property will appear as asset badge"
+                                disabledHoverText="Only Text or Number property types with allowed values defined can appear as an asset badge."
+                            />
+                        </StyledFormItem>
+                        <StyledFormItem name={['settings', 'showInColumnsTable']}>
+                            <Switch
+                                label="Show in Columns Table"
+                                size="sm"
+                                checked={formValues?.settings?.showInColumnsTable}
+                                onChange={(e) => handleDisplaySettingChange('showInColumnsTable', e.target.checked)}
+                                isDisabled={
+                                    !showInColumnsTable && (formValues?.settings?.isHidden || !hasColumnEntityType)
+                                }
+                                labelHoverText="If enabled, this property will appear as a column in the Columns table for Datasets"
+                                disabledHoverText="Property must apply to Columns in order to show in columns table."
+                                data-testid="structured-props-show-in-columns-table-switch"
+                            />
+                        </StyledFormItem>
+                    </TogglesContainer>
+                </Collapse.Panel>
+            </StyledCollapse>
+            {badgeProperty && (
+                <ConfirmationModal
+                    isOpen={showReplaceBadge}
+                    handleClose={handleReplaceClose}
+                    handleConfirm={() => {
+                        handleDisplaySettingChange('showAsAssetBadge', true);
+                        setShowReplaceBadge(false);
+                        updateBadgePropertyToOff();
+                    }}
+                    confirmButtonText="Update"
+                    modalTitle="Update Property"
+                    modalText={
+                        <p>
+                            <span>Another property </span>
+                            <Pill
+                                label={getDisplayName(badgeProperty)}
+                                size="sm"
+                                colorScheme="violet"
+                                clickable={false}
+                            />
+                            &nbsp;is already being shown on asset previews, but only one property is allowed at a time.
+                            Do you want to replace the current property? This will hide {getDisplayName(badgeProperty)}{' '}
+                            on all asset previews.
+                        </p>
+                    }
+                />
+            )}
+        </>
+    );
+};
+
+export default DisplayPreferences;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/EmptyStructuredProperties.tsx b/datahub-web-react/src/app/govern/structuredProperties/EmptyStructuredProperties.tsx
new file mode 100644
index 0000000000000..5f7f77b0582fd
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/EmptyStructuredProperties.tsx
@@ -0,0 +1,29 @@
+import { Text } from '@components';
+import EmptyFormsImage from '@src/images/empty-forms.svg?react';
+import React from 'react';
+import { EmptyContainer } from './styledComponents';
+
+interface Props {
+    isEmptySearch?: boolean;
+}
+
+const EmptyStructuredProperties = ({ isEmptySearch }: Props) => {
+    return (
+        <EmptyContainer>
+            {isEmptySearch ? (
+                <Text size="lg" color="gray" weight="bold">
+                    No search results!
+                </Text>
+            ) : (
+                <>
+                    <EmptyFormsImage />
+                    <Text size="md" color="gray" weight="bold">
+                        No properties yet!
+                    </Text>
+                </>
+            )}
+        </EmptyContainer>
+    );
+};
+
+export default EmptyStructuredProperties;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/RequiredAsterisk.tsx b/datahub-web-react/src/app/govern/structuredProperties/RequiredAsterisk.tsx
new file mode 100644
index 0000000000000..5842c363b3b31
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/RequiredAsterisk.tsx
@@ -0,0 +1,12 @@
+import { Text } from '@components';
+import React from 'react';
+
+const RequiredAsterisk = () => {
+    return (
+        <Text color="red" weight="bold">
+            *
+        </Text>
+    );
+};
+
+export default RequiredAsterisk;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/StructuredProperties.tsx b/datahub-web-react/src/app/govern/structuredProperties/StructuredProperties.tsx
new file mode 100644
index 0000000000000..ef1c495e0efa9
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/StructuredProperties.tsx
@@ -0,0 +1,115 @@
+import { Button, PageTitle, SearchBar, Tooltip } from '@components';
+import analytics, { EventType } from '@src/app/analytics';
+import { useUserContext } from '@src/app/context/useUserContext';
+import { useGetSearchResultsForMultipleQuery } from '@src/graphql/search.generated';
+import { EntityType, SearchResult, StructuredPropertyEntity } from '@src/types.generated';
+import React, { useState } from 'react';
+import StructuredPropsDrawer from './StructuredPropsDrawer';
+import StructuredPropsTable from './StructuredPropsTable';
+import { ButtonContainer, HeaderContainer, HeaderContent, PageContainer, TableContainer } from './styledComponents';
+import ViewStructuredPropsDrawer from './ViewStructuredPropsDrawer';
+
+const StructuredProperties = () => {
+    const [searchQuery, setSearchQuery] = useState<string>('');
+    const [isDrawerOpen, setIsDrawerOpen] = useState<boolean>(false);
+    const [isViewDrawerOpen, setIsViewDrawerOpen] = useState<boolean>(false);
+    const [selectedProperty, setSelectedProperty] = useState<SearchResult | undefined>();
+    const me = useUserContext();
+    const canEditProps = me.platformPrivileges?.manageStructuredProperties;
+
+    const handleSearch = (value) => {
+        setSearchQuery(value);
+    };
+
+    const inputs = {
+        types: [EntityType.StructuredProperty],
+        query: '',
+        start: 0,
+        count: 500,
+        searchFlags: { skipCache: true },
+    };
+
+    // Execute search
+    const { data, loading, refetch } = useGetSearchResultsForMultipleQuery({
+        variables: {
+            input: inputs,
+        },
+        fetchPolicy: 'cache-first',
+    });
+
+    const searchAcrossEntities = data?.searchAcrossEntities;
+    const noOfProperties = searchAcrossEntities?.searchResults?.length;
+    const badgeProperty = searchAcrossEntities?.searchResults?.find(
+        (prop) => (prop.entity as StructuredPropertyEntity).settings?.showAsAssetBadge,
+    )?.entity;
+
+    return (
+        <PageContainer>
+            <HeaderContainer>
+                <HeaderContent>
+                    <PageTitle
+                        title="Structured Properties"
+                        pillLabel={noOfProperties ? noOfProperties.toString() : undefined}
+                        subTitle="Create and manage custom properties for your data assets"
+                    />
+                </HeaderContent>
+                <Tooltip
+                    showArrow={false}
+                    title={
+                        !canEditProps
+                            ? 'Must have permission to manage structured properties. Ask your DataHub administrator.'
+                            : null
+                    }
+                >
+                    <ButtonContainer>
+                        <Button
+                            disabled={!canEditProps}
+                            icon="Add"
+                            data-testid="structured-props-create-button"
+                            onClick={() => {
+                                setIsDrawerOpen(true);
+                                analytics.event({ type: EventType.CreateStructuredPropertyClickEvent });
+                            }}
+                        >
+                            Create
+                        </Button>
+                    </ButtonContainer>
+                </Tooltip>
+            </HeaderContainer>
+            <SearchBar placeholder="Search" value={searchQuery} onChange={(value) => handleSearch(value)} />
+            <TableContainer>
+                <StructuredPropsTable
+                    searchQuery={searchQuery}
+                    data={data}
+                    loading={loading}
+                    setIsDrawerOpen={setIsDrawerOpen}
+                    setIsViewDrawerOpen={setIsViewDrawerOpen}
+                    setSelectedProperty={setSelectedProperty}
+                    selectedProperty={selectedProperty}
+                    inputs={inputs}
+                    searchAcrossEntities={searchAcrossEntities}
+                />
+            </TableContainer>
+            <StructuredPropsDrawer
+                isDrawerOpen={isDrawerOpen}
+                setIsDrawerOpen={setIsDrawerOpen}
+                refetch={refetch}
+                selectedProperty={selectedProperty}
+                setSelectedProperty={setSelectedProperty}
+                inputs={inputs}
+                searchAcrossEntities={searchAcrossEntities}
+                badgeProperty={badgeProperty as StructuredPropertyEntity}
+            />
+            {selectedProperty && (
+                <ViewStructuredPropsDrawer
+                    isViewDrawerOpen={isViewDrawerOpen}
+                    setIsViewDrawerOpen={setIsViewDrawerOpen}
+                    selectedProperty={selectedProperty}
+                    setSelectedProperty={setSelectedProperty}
+                />
+            )}
+        </PageContainer>
+    );
+};
+
+export default StructuredProperties;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsDrawer.tsx b/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsDrawer.tsx
new file mode 100644
index 0000000000000..debffeac7d583
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsDrawer.tsx
@@ -0,0 +1,397 @@
+import { LoadingOutlined } from '@ant-design/icons';
+import { useApolloClient } from '@apollo/client';
+import { Button, Text } from '@src/alchemy-components';
+import analytics, { EventType } from '@src/app/analytics';
+import { useUserContext } from '@src/app/context/useUserContext';
+import { showToastMessage, ToastType } from '@src/app/sharedV2/toastMessageUtils';
+import {
+    useCreateStructuredPropertyMutation,
+    useUpdateStructuredPropertyMutation,
+} from '@src/graphql/structuredProperties.generated';
+import {
+    AllowedValue,
+    PropertyCardinality,
+    SearchAcrossEntitiesInput,
+    SearchResult,
+    SearchResults,
+    StructuredPropertyEntity,
+    UpdateStructuredPropertyInput,
+} from '@src/types.generated';
+import { Form } from 'antd';
+import { Tooltip } from '@components';
+import React, { useEffect, useState } from 'react';
+import AllowedValuesDrawer from './AllowedValuesDrawer';
+import { updatePropertiesList } from './cacheUtils';
+import StructuredPropsForm from './StructuredPropsForm';
+import {
+    DrawerHeader,
+    FooterContainer,
+    StyledDrawer,
+    StyledIcon,
+    StyledSpin,
+    TitleContainer,
+} from './styledComponents';
+import useStructuredProp from './useStructuredProp';
+import {
+    getDisplayName,
+    getNewAllowedTypes,
+    getNewAllowedValues,
+    getNewEntityTypes,
+    getStringOrNumberValueField,
+    getValueType,
+    PropValueField,
+    StructuredProp,
+    valueTypes,
+} from './utils';
+
+interface Props {
+    isDrawerOpen: boolean;
+    setIsDrawerOpen: React.Dispatch<React.SetStateAction<boolean>>;
+    selectedProperty?: SearchResult;
+    setSelectedProperty: React.Dispatch<React.SetStateAction<SearchResult | undefined>>;
+    refetch: () => void;
+    inputs: SearchAcrossEntitiesInput;
+    searchAcrossEntities?: SearchResults | null;
+    badgeProperty?: StructuredPropertyEntity;
+}
+
+const StructuredPropsDrawer = ({
+    isDrawerOpen,
+    setIsDrawerOpen,
+    selectedProperty,
+    setSelectedProperty,
+    refetch,
+    inputs,
+    searchAcrossEntities,
+    badgeProperty,
+}: Props) => {
+    const [form] = Form.useForm();
+    const [valuesForm] = Form.useForm();
+    const me = useUserContext();
+    const canEditProps = me.platformPrivileges?.manageStructuredProperties;
+
+    const [createStructuredProperty] = useCreateStructuredPropertyMutation();
+    const [updateStructuredProperty] = useUpdateStructuredPropertyMutation();
+    const client = useApolloClient();
+
+    const [cardinality, setCardinality] = useState<PropertyCardinality>(PropertyCardinality.Single);
+    const [formValues, setFormValues] = useState<StructuredProp>();
+    const [selectedValueType, setSelectedValueType] = useState<string>('');
+    const [allowedValues, setAllowedValues] = useState<AllowedValue[] | undefined>([]);
+    const [isLoading, setIsLoading] = useState<boolean>(false);
+    const [valueField, setValueField] = useState<PropValueField>('stringValue');
+
+    const [showAllowedValuesDrawer, setShowAllowedValuesDrawer] = useState<boolean>(false);
+
+    const { handleTypeUpdate } = useStructuredProp({
+        selectedProperty,
+        form,
+        setFormValues,
+        setCardinality,
+        setSelectedValueType,
+    });
+
+    const isEditMode = !!selectedProperty;
+
+    const clearValues = () => {
+        form.resetFields();
+        setIsDrawerOpen(false);
+        setSelectedProperty(undefined);
+        setFormValues(undefined);
+        setSelectedValueType('');
+        setAllowedValues(undefined);
+        setCardinality(PropertyCardinality.Single);
+        setShowAllowedValuesDrawer(false);
+        valuesForm.resetFields();
+    };
+
+    const handleClose = () => {
+        clearValues();
+    };
+
+    const showErrorMessage = () => {
+        showToastMessage(ToastType.ERROR, `Failed to ${isEditMode ? 'update' : 'create'} structured property.`, 3);
+    };
+
+    const showSuccessMessage = () => {
+        showToastMessage(ToastType.SUCCESS, `Structured property ${isEditMode ? 'updated' : 'created'}!`, 3);
+    };
+
+    const handleSubmit = () => {
+        if (isEditMode) {
+            form.validateFields().then(() => {
+                const updateValues = {
+                    ...form.getFieldsValue(),
+                    allowedValues,
+                };
+
+                const editInput: UpdateStructuredPropertyInput = {
+                    urn: selectedProperty.entity.urn,
+                    displayName: updateValues.displayName,
+                    description: updateValues.description,
+                    typeQualifier: {
+                        newAllowedTypes: getNewAllowedTypes(
+                            selectedProperty.entity as StructuredPropertyEntity,
+                            updateValues,
+                        ),
+                    },
+                    newEntityTypes: getNewEntityTypes(
+                        selectedProperty.entity as StructuredPropertyEntity,
+                        updateValues,
+                    ),
+                    newAllowedValues: getNewAllowedValues(
+                        selectedProperty.entity as StructuredPropertyEntity,
+                        updateValues,
+                    ),
+                    setCardinalityAsMultiple: cardinality === PropertyCardinality.Multiple,
+                    settings: {
+                        isHidden: updateValues.settings?.isHidden ?? false,
+                        showInSearchFilters: updateValues.settings?.showInSearchFilters ?? false,
+                        showAsAssetBadge: updateValues.settings?.showAsAssetBadge ?? false,
+                        showInAssetSummary: updateValues.settings?.showInAssetSummary ?? false,
+                        showInColumnsTable: updateValues.settings?.showInColumnsTable ?? false,
+                    },
+                };
+
+                setIsLoading(true);
+                updateStructuredProperty({
+                    variables: {
+                        input: editInput,
+                    },
+                })
+                    .then(() => {
+                        analytics.event({
+                            type: EventType.EditStructuredPropertyEvent,
+                            propertyUrn: selectedProperty.entity.urn,
+                            propertyType:
+                                valueTypes.find((valType) => valType.value === form.getFieldValue('valueType'))?.urn ||
+                                '',
+                            appliesTo: form.getFieldValue('entityTypes'),
+                            qualifiedName: form.getFieldValue('qualifiedName'),
+                            allowedAssetTypes: form.getFieldValue(['typeQualifier', 'allowedTypes']),
+                            allowedValues: form.getFieldValue('allowedValues'),
+                            cardinality,
+                            isHidden: form.getFieldValue(['settings', 'isHidden']) ?? false,
+                            showInSearchFilters: form.getFieldValue(['settings', 'showInSearchFilters']) ?? false,
+                            showAsAssetBadge: form.getFieldValue(['settings', 'showAsAssetBadge']) ?? false,
+                            showInAssetSummary: form.getFieldValue(['settings', 'showInAssetSummary']) ?? false,
+                            showInColumnsTable: form.getFieldValue(['settings', 'showInColumnsTable']) ?? false,
+                        });
+                        refetch();
+                        showSuccessMessage();
+                    })
+                    .catch(() => {
+                        showErrorMessage();
+                    })
+                    .finally(() => {
+                        setIsLoading(false);
+                        clearValues();
+                    });
+            });
+        } else {
+            form.validateFields().then(() => {
+                const createInput = {
+                    ...form.getFieldsValue(),
+                    qualifiedName: form.getFieldValue('qualifiedName') || undefined,
+                    valueType: valueTypes.find((type) => type.value === form.getFieldValue('valueType'))?.urn,
+                    allowedValues,
+                    cardinality,
+                    settings: {
+                        isHidden: form.getFieldValue(['settings', 'isHidden']) ?? false,
+                        showInSearchFilters: form.getFieldValue(['settings', 'showInSearchFilters']) ?? false,
+                        showAsAssetBadge: form.getFieldValue(['settings', 'showAsAssetBadge']) ?? false,
+                        showInAssetSummary: form.getFieldValue(['settings', 'showInAssetSummary']) ?? false,
+                        showInColumnsTable: form.getFieldValue(['settings', 'showInColumnsTable']) ?? false,
+                    },
+                };
+
+                setIsLoading(true);
+                createStructuredProperty({
+                    variables: {
+                        input: createInput,
+                    },
+                })
+                    .then((res) => {
+                        analytics.event({
+                            type: EventType.CreateStructuredPropertyEvent,
+                            propertyType:
+                                valueTypes.find((valType) => valType.value === form.getFieldValue('valueType'))?.urn ||
+                                '',
+                            appliesTo: form.getFieldValue('entityTypes'),
+                            qualifiedName: form.getFieldValue('qualifiedName'),
+                            allowedAssetTypes: form.getFieldValue(['typeQualifier', 'allowedTypes']),
+                            allowedValues: form.getFieldValue('allowedValues'),
+                            cardinality,
+                            isHidden: form.getFieldValue(['settings', 'isHidden']) ?? false,
+                            showInSearchFilters: form.getFieldValue(['settings', 'showInSearchFilters']) ?? false,
+                            showAsAssetBadge: form.getFieldValue(['settings', 'showAsAssetBadge']) ?? false,
+                            showInAssetSummary: form.getFieldValue(['settings', 'showInAssetSummary']) ?? false,
+                            showInColumnsTable: form.getFieldValue(['settings', 'showInColumnsTable']) ?? false,
+                        });
+
+                        showSuccessMessage();
+                        updatePropertiesList(client, inputs, res.data?.createStructuredProperty, searchAcrossEntities);
+                    })
+                    .catch(() => {
+                        showErrorMessage();
+                    })
+                    .finally(() => {
+                        setIsLoading(false);
+                        clearValues();
+                    });
+            });
+        }
+    };
+
+    useEffect(() => {
+        if (selectedProperty) {
+            const entity = selectedProperty.entity as StructuredPropertyEntity;
+            const typeValue = getValueType(
+                entity.definition.valueType.urn,
+                entity.definition.cardinality || PropertyCardinality.Single,
+            );
+
+            const values: StructuredProp = {
+                displayName: getDisplayName(entity),
+                description: entity.definition.description,
+                qualifiedName: entity.definition.qualifiedName,
+                valueType: typeValue,
+                entityTypes: entity.definition.entityTypes.map((entityType) => entityType.urn),
+                typeQualifier: {
+                    allowedTypes: entity.definition.typeQualifier?.allowedTypes?.map((entityType) => entityType.urn),
+                },
+                immutable: entity.definition.immutable,
+                settings: entity.settings,
+            };
+
+            setFormValues(values);
+            if (typeValue) handleTypeUpdate(typeValue);
+            form.setFieldsValue(values);
+        } else {
+            setFormValues(undefined);
+            form.resetFields();
+            setSelectedValueType('');
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [selectedProperty, form]);
+
+    useEffect(() => {
+        const entity = selectedProperty?.entity as StructuredPropertyEntity;
+        const field = getStringOrNumberValueField(selectedValueType);
+        setValueField(field);
+        const allowedList = entity?.definition?.allowedValues?.map((item) => {
+            return {
+                [field]: item.value[field],
+                description: item.description,
+            } as AllowedValue;
+        });
+        setAllowedValues(allowedList);
+    }, [selectedProperty, selectedValueType, setAllowedValues]);
+
+    const handleUpdateAllowedValues = () => {
+        valuesForm.validateFields().then(() => {
+            setAllowedValues(valuesForm.getFieldValue('allowedValues'));
+            form.setFieldValue('allowedValues', valuesForm.getFieldValue('allowedValues'));
+            setShowAllowedValuesDrawer(false);
+        });
+    };
+
+    return (
+        <StyledDrawer
+            open={isDrawerOpen}
+            closable={false}
+            width={480}
+            title={
+                <DrawerHeader>
+                    {showAllowedValuesDrawer ? (
+                        <TitleContainer>
+                            <StyledIcon
+                                icon="ArrowBack"
+                                color="gray"
+                                size="3xl"
+                                onClick={() => setShowAllowedValuesDrawer(false)}
+                            />
+                            <Text color="gray" weight="bold" size="lg">
+                                Allowed Values
+                            </Text>
+                        </TitleContainer>
+                    ) : (
+                        <Text color="gray" weight="bold" size="lg">
+                            {`${isEditMode ? 'Edit' : 'Create'} Structured Property`}
+                        </Text>
+                    )}
+                    <StyledIcon icon="Close" color="gray" onClick={handleClose} />
+                </DrawerHeader>
+            }
+            footer={
+                <Tooltip
+                    showArrow={false}
+                    title={
+                        !canEditProps
+                            ? 'Must have permission to manage structured properties. Ask your DataHub administrator.'
+                            : null
+                    }
+                >
+                    <FooterContainer>
+                        {showAllowedValuesDrawer ? (
+                            <Button
+                                style={{ display: 'block', width: '100%' }}
+                                onClick={handleUpdateAllowedValues}
+                                isDisabled={!canEditProps}
+                            >
+                                Update Allowed Values
+                            </Button>
+                        ) : (
+                            <Button
+                                style={{ display: 'block', width: '100%' }}
+                                onClick={handleSubmit}
+                                isDisabled={isLoading || !canEditProps}
+                                data-testid="structured-props-create-update-button"
+                            >
+                                {isEditMode ? 'Update' : 'Create'}
+                            </Button>
+                        )}
+                    </FooterContainer>
+                </Tooltip>
+            }
+            destroyOnClose
+        >
+            <StyledSpin spinning={isLoading} indicator={<LoadingOutlined />}>
+                {showAllowedValuesDrawer ? (
+                    <>
+                        <AllowedValuesDrawer
+                            showAllowedValuesDrawer={showAllowedValuesDrawer}
+                            propType={valueField}
+                            allowedValues={allowedValues}
+                            isEditMode={isEditMode}
+                            noOfExistingValues={
+                                (selectedProperty?.entity as StructuredPropertyEntity)?.definition?.allowedValues
+                                    ?.length || 0
+                            }
+                            form={valuesForm}
+                        />
+                    </>
+                ) : (
+                    <StructuredPropsForm
+                        selectedProperty={selectedProperty}
+                        form={form}
+                        formValues={formValues}
+                        setFormValues={setFormValues}
+                        setCardinality={setCardinality}
+                        isEditMode={isEditMode}
+                        selectedValueType={selectedValueType}
+                        setSelectedValueType={setSelectedValueType}
+                        allowedValues={allowedValues}
+                        valueField={valueField}
+                        setShowAllowedValuesDrawer={setShowAllowedValuesDrawer}
+                        refetchProperties={refetch}
+                        badgeProperty={badgeProperty}
+                    />
+                )}
+            </StyledSpin>
+        </StyledDrawer>
+    );
+};
+
+export default StructuredPropsDrawer;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsForm.tsx b/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsForm.tsx
new file mode 100644
index 0000000000000..5159936722884
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsForm.tsx
@@ -0,0 +1,139 @@
+import { Icon, Input, SimpleSelect, TextArea } from '@src/alchemy-components';
+import { AllowedValue, PropertyCardinality, SearchResult, StructuredPropertyEntity } from '@src/types.generated';
+import { Form, FormInstance } from 'antd';
+import { Tooltip } from '@components';
+import React from 'react';
+import AdvancedOptions from './AdvancedOptions';
+import RequiredAsterisk from './RequiredAsterisk';
+import DisplayPreferences from './DisplayPreferences';
+import StructuredPropsFormSection from './StructuredPropsFormSection';
+import { FieldLabel, FlexContainer, GridFormItem, RowContainer } from './styledComponents';
+import useStructuredProp from './useStructuredProp';
+import { PropValueField, StructuredProp, valueTypes } from './utils';
+
+interface Props {
+    selectedProperty: SearchResult | undefined;
+    form: FormInstance;
+    formValues: StructuredProp | undefined;
+    setFormValues: React.Dispatch<React.SetStateAction<StructuredProp | undefined>>;
+    setCardinality: React.Dispatch<React.SetStateAction<PropertyCardinality>>;
+    isEditMode: boolean;
+    selectedValueType: string;
+    setSelectedValueType: React.Dispatch<React.SetStateAction<string>>;
+    allowedValues: AllowedValue[] | undefined;
+    valueField: PropValueField;
+    setShowAllowedValuesDrawer: React.Dispatch<React.SetStateAction<boolean>>;
+    refetchProperties: () => void;
+    badgeProperty?: StructuredPropertyEntity;
+}
+
+const StructuredPropsForm = ({
+    selectedProperty,
+    form,
+    formValues,
+    setFormValues,
+    isEditMode,
+    setCardinality,
+    selectedValueType,
+    setSelectedValueType,
+    allowedValues,
+    valueField,
+    setShowAllowedValuesDrawer,
+    refetchProperties,
+    badgeProperty,
+}: Props) => {
+    const { handleTypeUpdate, handleDisplaySettingChange } = useStructuredProp({
+        selectedProperty,
+        form,
+        setFormValues,
+        setCardinality,
+        setSelectedValueType,
+    });
+
+    return (
+        <Form form={form}>
+            <Form.Item
+                name="displayName"
+                rules={[
+                    {
+                        required: true,
+                        message: 'Please enter the name',
+                    },
+                ]}
+            >
+                <Input label="Name" placeholder="Provide a name" isRequired data-testid="structured-props-input-name" />
+            </Form.Item>
+            <Form.Item name="description">
+                <TextArea
+                    label="Description"
+                    placeholder="Provide a description"
+                    data-testid="structured-props-input-description"
+                />
+            </Form.Item>
+            <RowContainer>
+                <FieldLabel>
+                    <FlexContainer>
+                        Property Type
+                        <RequiredAsterisk />
+                        <Tooltip title="The allowed value type of the property" showArrow={false}>
+                            <Icon icon="Info" color="violet" size="lg" />
+                        </Tooltip>
+                    </FlexContainer>
+                </FieldLabel>
+
+                <Tooltip
+                    title={isEditMode && 'Once a property is created, its type cannot be changed'}
+                    showArrow={false}
+                >
+                    <GridFormItem
+                        name="valueType"
+                        rules={[
+                            {
+                                required: true,
+                                message: 'Please select the property type',
+                            },
+                        ]}
+                    >
+                        <SimpleSelect
+                            onUpdate={(values: any) => {
+                                handleTypeUpdate(values[0]);
+                            }}
+                            placeholder="Select Property Type"
+                            options={valueTypes}
+                            values={formValues?.valueType ? [formValues?.valueType] : undefined}
+                            isDisabled={isEditMode}
+                            showDescriptions
+                            data-testid="structured-props-select-input-type"
+                            optionListTestId="structured-props-property-type-options-list"
+                        />
+                    </GridFormItem>
+                </Tooltip>
+            </RowContainer>
+
+            <StructuredPropsFormSection
+                selectedProperty={selectedProperty}
+                form={form}
+                formValues={formValues}
+                setFormValues={setFormValues}
+                isEditMode={isEditMode}
+                setCardinality={setCardinality}
+                selectedValueType={selectedValueType}
+                setSelectedValueType={setSelectedValueType}
+                allowedValues={allowedValues}
+                valueField={valueField}
+                setShowAllowedValuesDrawer={setShowAllowedValuesDrawer}
+            />
+            <DisplayPreferences
+                formValues={formValues}
+                handleDisplaySettingChange={handleDisplaySettingChange}
+                selectedValueType={selectedValueType}
+                refetchProperties={refetchProperties}
+                badgeProperty={badgeProperty}
+                allowedValues={allowedValues}
+            />
+            <AdvancedOptions isEditMode={isEditMode} />
+        </Form>
+    );
+};
+
+export default StructuredPropsForm;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsFormSection.tsx b/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsFormSection.tsx
new file mode 100644
index 0000000000000..82875c8f58a4d
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsFormSection.tsx
@@ -0,0 +1,176 @@
+import { Icon, SimpleSelect, Text, Tooltip } from '@src/alchemy-components';
+import { AllowedValue, PropertyCardinality, SearchResult } from '@src/types.generated';
+import { Form, FormInstance } from 'antd';
+import React from 'react';
+import AllowedValuesField from './AllowedValuesField';
+import RequiredAsterisk from './RequiredAsterisk';
+import { FieldLabel, FlexContainer, RowContainer, SubTextContainer } from './styledComponents';
+import useStructuredProp from './useStructuredProp';
+import {
+    APPLIES_TO_ENTITIES,
+    isEntityTypeSelected,
+    PropValueField,
+    SEARCHABLE_ENTITY_TYPES,
+    StructuredProp,
+} from './utils';
+
+interface Props {
+    selectedProperty: SearchResult | undefined;
+    form: FormInstance;
+    formValues: StructuredProp | undefined;
+    setFormValues: React.Dispatch<React.SetStateAction<StructuredProp | undefined>>;
+    setCardinality: React.Dispatch<React.SetStateAction<PropertyCardinality>>;
+    isEditMode: boolean;
+    selectedValueType: string;
+    setSelectedValueType: React.Dispatch<React.SetStateAction<string>>;
+    allowedValues: AllowedValue[] | undefined;
+    valueField: PropValueField;
+    setShowAllowedValuesDrawer: React.Dispatch<React.SetStateAction<boolean>>;
+}
+
+const StructuredPropsFormSection = ({
+    selectedProperty,
+    form,
+    formValues,
+    setFormValues,
+    isEditMode,
+    setCardinality,
+    selectedValueType,
+    setSelectedValueType,
+    allowedValues,
+    valueField,
+    setShowAllowedValuesDrawer,
+}: Props) => {
+    const {
+        handleSelectChange,
+        handleSelectUpdateChange,
+        getEntitiesListOptions,
+        disabledEntityTypeValues,
+        disabledTypeQualifierValues,
+    } = useStructuredProp({
+        selectedProperty,
+        form,
+        setFormValues,
+        setCardinality,
+        setSelectedValueType,
+    });
+
+    return (
+        <>
+            {!(isEditMode && !allowedValues) && (
+                <AllowedValuesField
+                    selectedValueType={selectedValueType}
+                    allowedValues={allowedValues}
+                    valueField={valueField}
+                    setShowAllowedValuesDrawer={setShowAllowedValuesDrawer}
+                />
+            )}
+            {isEntityTypeSelected(selectedValueType) && (
+                <RowContainer>
+                    <FieldLabel>
+                        <FlexContainer>
+                            Allowed Entity Types
+                            <Tooltip
+                                title="Choose the types of entities that are allowed as values for this property"
+                                showArrow={false}
+                            >
+                                <Icon icon="Info" color="violet" size="lg" />
+                            </Tooltip>
+                        </FlexContainer>
+                        {isEditMode && (
+                            <SubTextContainer>
+                                <Text size="sm" weight="medium">
+                                    <Tooltip
+                                        title="Once a property is created, entity types cannot be removed"
+                                        showArrow={false}
+                                    >
+                                        (Add-only)
+                                    </Tooltip>
+                                </Text>
+                            </SubTextContainer>
+                        )}
+                    </FieldLabel>
+                    <Tooltip
+                        title={
+                            !formValues?.typeQualifier?.allowedTypes?.length &&
+                            'Any entity type will be accepted as a value'
+                        }
+                        showArrow={false}
+                    >
+                        <Form.Item name={['typeQualifier', 'allowedTypes']}>
+                            <SimpleSelect
+                                options={getEntitiesListOptions(SEARCHABLE_ENTITY_TYPES)}
+                                onUpdate={(values) =>
+                                    isEditMode
+                                        ? handleSelectUpdateChange(['typeQualifier', 'allowedTypes'], values)
+                                        : handleSelectChange(['typeQualifier', 'allowedTypes'], values)
+                                }
+                                placeholder="Any"
+                                isMultiSelect
+                                values={formValues?.typeQualifier?.allowedTypes}
+                                disabledValues={disabledTypeQualifierValues}
+                                width="full"
+                            />
+                        </Form.Item>
+                    </Tooltip>
+                </RowContainer>
+            )}
+            <RowContainer>
+                <FieldLabel>
+                    <FlexContainer>
+                        Applies to
+                        <RequiredAsterisk />
+                        <Tooltip
+                            title="Select the types of entities that this property can be added to"
+                            showArrow={false}
+                        >
+                            <Icon icon="Info" color="violet" size="lg" />
+                        </Tooltip>
+                    </FlexContainer>
+                    {isEditMode && (
+                        <SubTextContainer>
+                            <Text size="sm" weight="medium">
+                                <Tooltip
+                                    title="Once a property is created entity types cannot be removed"
+                                    showArrow={false}
+                                >
+                                    (Add-only)
+                                </Tooltip>
+                            </Text>
+                        </SubTextContainer>
+                    )}
+                </FieldLabel>
+
+                <Form.Item
+                    name="entityTypes"
+                    rules={[
+                        {
+                            required: true,
+                            message: 'Please select asset types this applies to',
+                        },
+                    ]}
+                >
+                    <SimpleSelect
+                        options={getEntitiesListOptions(APPLIES_TO_ENTITIES)}
+                        onUpdate={(values) =>
+                            isEditMode
+                                ? handleSelectUpdateChange('entityTypes', values)
+                                : handleSelectChange('entityTypes', values)
+                        }
+                        placeholder="Select Entity Types"
+                        isMultiSelect
+                        values={formValues?.entityTypes ? formValues?.entityTypes : undefined}
+                        disabledValues={disabledEntityTypeValues}
+                        width="full"
+                        showSelectAll
+                        selectAllLabel="All Asset Types"
+                        data-testid="structured-props-select-input-applies-to"
+                        optionListTestId="applies-to-options-list"
+                    />
+                </Form.Item>
+            </RowContainer>
+        </>
+    );
+};
+
+export default StructuredPropsFormSection;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsTable.tsx b/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsTable.tsx
new file mode 100644
index 0000000000000..10c42c0925eea
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/StructuredPropsTable.tsx
@@ -0,0 +1,376 @@
+import { useApolloClient } from '@apollo/client';
+import { Icon, Pill, Table, Text, Tooltip } from '@components';
+import { AlignmentOptions } from '@src/alchemy-components/theme/config';
+import analytics, { EventType } from '@src/app/analytics';
+import { useUserContext } from '@src/app/context/useUserContext';
+import { HoverEntityTooltip } from '@src/app/recommendations/renderer/component/HoverEntityTooltip';
+import { CustomAvatar } from '@src/app/shared/avatar';
+import { toLocalDateString, toRelativeTimeString } from '@src/app/shared/time/timeUtils';
+import { ConfirmationModal } from '@src/app/sharedV2/modals/ConfirmationModal';
+import { showToastMessage, ToastType } from '@src/app/sharedV2/toastMessageUtils';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { GetSearchResultsForMultipleQuery } from '@src/graphql/search.generated';
+import { useDeleteStructuredPropertyMutation } from '@src/graphql/structuredProperties.generated';
+import TableIcon from '@src/images/table-icon.svg?react';
+import {
+    Entity,
+    EntityType,
+    SearchAcrossEntitiesInput,
+    SearchResult,
+    SearchResults,
+    StructuredPropertyEntity,
+} from '@src/types.generated';
+import { Dropdown } from 'antd';
+import React, { useState } from 'react';
+import Highlight from 'react-highlighter';
+import { Link } from 'react-router-dom';
+import { removeFromPropertiesList } from './cacheUtils';
+import EmptyStructuredProperties from './EmptyStructuredProperties';
+import {
+    CardIcons,
+    CreatedByContainer,
+    DataContainer,
+    IconContainer,
+    MenuItem,
+    NameColumn,
+    PillContainer,
+    PillsContainer,
+    PropDescription,
+    PropName,
+} from './styledComponents';
+import { getDisplayName } from './utils';
+
+interface Props {
+    searchQuery: string;
+    data: GetSearchResultsForMultipleQuery | undefined;
+    loading: boolean;
+    setIsDrawerOpen: React.Dispatch<React.SetStateAction<boolean>>;
+    setIsViewDrawerOpen: React.Dispatch<React.SetStateAction<boolean>>;
+    selectedProperty?: SearchResult;
+    setSelectedProperty: React.Dispatch<React.SetStateAction<SearchResult | undefined>>;
+    inputs: SearchAcrossEntitiesInput;
+    searchAcrossEntities?: SearchResults | null;
+}
+
+const StructuredPropsTable = ({
+    searchQuery,
+    data,
+    loading,
+    setIsDrawerOpen,
+    setIsViewDrawerOpen,
+    selectedProperty,
+    setSelectedProperty,
+    inputs,
+    searchAcrossEntities,
+}: Props) => {
+    const entityRegistry = useEntityRegistry();
+    const client = useApolloClient();
+    const me = useUserContext();
+    const canEditProps = me.platformPrivileges?.manageStructuredProperties;
+
+    const structuredProperties = data?.searchAcrossEntities?.searchResults || [];
+
+    // Filter the table data based on the search query
+    const filteredProperties = structuredProperties
+        .filter((prop: any) => prop.entity.definition?.displayName?.toLowerCase().includes(searchQuery.toLowerCase()))
+        .sort(
+            (propA, propB) =>
+                ((propB.entity as StructuredPropertyEntity).definition.created?.time || 0) -
+                ((propA.entity as StructuredPropertyEntity).definition.created?.time || 0),
+        );
+
+    const [deleteStructuredProperty] = useDeleteStructuredPropertyMutation();
+
+    const [showConfirmDelete, setShowConfirmDelete] = useState<boolean>(false);
+
+    const handleDeleteProperty = (property) => {
+        const deleteEntity = property.entity as StructuredPropertyEntity;
+        showToastMessage(ToastType.LOADING, 'Deleting structured property', 1);
+        deleteStructuredProperty({
+            variables: {
+                input: {
+                    urn: deleteEntity.urn,
+                },
+            },
+        })
+            .then(() => {
+                analytics.event({
+                    type: EventType.DeleteStructuredPropertyEvent,
+                    propertyUrn: property.entity.urn,
+                    propertyType: deleteEntity.definition.valueType.urn,
+                    appliesTo: deleteEntity.definition.entityTypes.map((type) => type.urn),
+                    qualifiedName: deleteEntity.definition.qualifiedName,
+                    showInFilters: deleteEntity.settings?.showInSearchFilters,
+                    allowedAssetTypes: deleteEntity.definition.typeQualifier?.allowedTypes?.map(
+                        (allowedType) => allowedType.urn,
+                    ),
+                    allowedValues: deleteEntity.definition.allowedValues || undefined,
+                    cardinality: deleteEntity.definition.cardinality || undefined,
+                    isHidden: deleteEntity.settings?.isHidden ?? false,
+                    showInSearchFilters: deleteEntity.settings?.showInSearchFilters ?? false,
+                    showAsAssetBadge: deleteEntity.settings?.showAsAssetBadge ?? false,
+                    showInAssetSummary: deleteEntity.settings?.showInAssetSummary ?? false,
+                    showInColumnsTable: deleteEntity.settings?.showInColumnsTable ?? false,
+                });
+                showToastMessage(ToastType.SUCCESS, 'Structured property deleted successfully!', 3);
+                removeFromPropertiesList(client, inputs, property.entity.urn, searchAcrossEntities);
+            })
+            .catch(() => {
+                showToastMessage(ToastType.ERROR, 'Failed to delete structured property', 3);
+            });
+
+        setShowConfirmDelete(false);
+        setSelectedProperty(undefined);
+    };
+
+    const handleDeleteClose = () => {
+        setShowConfirmDelete(false);
+        setSelectedProperty(undefined);
+    };
+
+    if (!loading && !filteredProperties.length) {
+        return <EmptyStructuredProperties isEmptySearch={!!structuredProperties.length} />;
+    }
+
+    const columns = [
+        {
+            title: 'Name',
+            key: 'name',
+            render: (record) => {
+                return (
+                    <NameColumn>
+                        <IconContainer>
+                            <TableIcon color="#705EE4" />
+                        </IconContainer>
+                        <DataContainer>
+                            <PropName
+                                ellipsis={{ tooltip: getDisplayName(record.entity) }}
+                                onClick={() => {
+                                    if (canEditProps) setIsDrawerOpen(true);
+                                    else setIsViewDrawerOpen(true);
+
+                                    setSelectedProperty(record);
+                                    analytics.event({
+                                        type: EventType.ViewStructuredPropertyEvent,
+                                        propertyUrn: record.entity.urn,
+                                    });
+                                }}
+                            >
+                                <Highlight search={searchQuery}>{getDisplayName(record.entity)}</Highlight>
+                            </PropName>
+                            <PropDescription ellipsis>{record.entity.definition.description}</PropDescription>
+                        </DataContainer>
+                    </NameColumn>
+                );
+            },
+            width: '580px',
+            sorter: (sourceA, sourceB) => {
+                return getDisplayName(sourceA.entity).localeCompare(getDisplayName(sourceB.entity));
+            },
+        },
+        {
+            title: 'Entity Types',
+            key: 'entityTypes',
+            width: '270px',
+            render: (record) => {
+                const types = record.entity.definition.entityTypes;
+                const maxTypesToShow = 2;
+                const overflowCount = types.length - maxTypesToShow;
+
+                return (
+                    <PillsContainer>
+                        {types.slice(0, maxTypesToShow).map((entityType) => {
+                            const typeName = entityRegistry.getEntityName(entityType.info.type);
+                            return (
+                                <PillContainer>{typeName && <Pill label={typeName} clickable={false} />}</PillContainer>
+                            );
+                        })}
+                        {overflowCount > 0 && (
+                            <Tooltip
+                                title={types
+                                    .slice(maxTypesToShow)
+                                    .map((eType) => {
+                                        const name = entityRegistry.getEntityName(eType.info.type);
+                                        return name;
+                                    })
+                                    .join(', ')}
+                                showArrow={false}
+                            >
+                                <>
+                                    <Text>{`+${overflowCount}`}</Text>
+                                </>
+                            </Tooltip>
+                        )}
+                    </PillsContainer>
+                );
+            },
+        },
+        {
+            title: 'Creation Date',
+            key: 'creationDate',
+            render: (record) => {
+                const createdTime = record.entity.definition.created?.time;
+                return (
+                    <Tooltip title={toLocalDateString(createdTime)} showArrow={false}>
+                        {createdTime ? toRelativeTimeString(createdTime) : '-'}
+                    </Tooltip>
+                );
+            },
+            sorter: (sourceA, sourceB) => {
+                const timeA = sourceA.entity.definition.created?.time || Number.MAX_SAFE_INTEGER;
+                const timeB = sourceB.entity.definition.created?.time || Number.MAX_SAFE_INTEGER;
+
+                return timeA - timeB;
+            },
+        },
+
+        {
+            title: 'Created By',
+            key: 'createdBy',
+            render: (record) => {
+                const createdByUser = record.entity.definition?.created?.actor;
+                const name = createdByUser && entityRegistry.getDisplayName(EntityType.CorpUser, createdByUser);
+                const avatarUrl = createdByUser?.editableProperties?.pictureLink || undefined;
+
+                return (
+                    <>
+                        {createdByUser && (
+                            <HoverEntityTooltip entity={createdByUser as Entity}>
+                                <Link
+                                    to={`${entityRegistry.getEntityUrl(
+                                        EntityType.CorpUser,
+                                        (createdByUser as Entity).urn,
+                                    )}`}
+                                >
+                                    <CreatedByContainer>
+                                        <CustomAvatar size={20} name={name} photoUrl={avatarUrl} hideTooltip />
+                                        <Text color="gray" size="sm">
+                                            {name}
+                                        </Text>
+                                    </CreatedByContainer>
+                                </Link>
+                            </HoverEntityTooltip>
+                        )}
+                    </>
+                );
+            },
+            sorter: (sourceA, sourceB) => {
+                const createdByUserA = sourceA.entity.definition?.created?.actor;
+                const nameA = createdByUserA && entityRegistry.getDisplayName(EntityType.CorpUser, createdByUserA);
+                const createdByUserB = sourceB.entity.definition?.created?.actor;
+                const nameB = createdByUserB && entityRegistry.getDisplayName(EntityType.CorpUser, createdByUserB);
+
+                return nameA?.localeCompare(nameB);
+            },
+        },
+        {
+            title: '',
+            key: 'actions',
+            alignment: 'right' as AlignmentOptions,
+            render: (record) => {
+                const items = [
+                    {
+                        key: '0',
+                        label: (
+                            <MenuItem
+                                onClick={() => {
+                                    setIsViewDrawerOpen(true);
+                                    setSelectedProperty(record);
+                                    analytics.event({
+                                        type: EventType.ViewStructuredPropertyEvent,
+                                        propertyUrn: record.entity.urn,
+                                    });
+                                }}
+                            >
+                                View
+                            </MenuItem>
+                        ),
+                    },
+                    {
+                        key: '1',
+                        disabled: !canEditProps,
+                        label: (
+                            <Tooltip
+                                showArrow={false}
+                                title={
+                                    !canEditProps
+                                        ? 'Must have permission to manage structured properties. Ask your DataHub administrator.'
+                                        : null
+                                }
+                            >
+                                <MenuItem
+                                    onClick={() => {
+                                        if (canEditProps) {
+                                            setIsDrawerOpen(true);
+                                            setSelectedProperty(record);
+                                            analytics.event({
+                                                type: EventType.ViewStructuredPropertyEvent,
+                                                propertyUrn: record.entity.urn,
+                                            });
+                                        }
+                                    }}
+                                >
+                                    Edit
+                                </MenuItem>
+                            </Tooltip>
+                        ),
+                    },
+                    {
+                        key: '2',
+                        disabled: !canEditProps,
+                        label: (
+                            <Tooltip
+                                showArrow={false}
+                                title={
+                                    !canEditProps
+                                        ? 'Must have permission to manage structured properties. Ask your DataHub administrator.'
+                                        : null
+                                }
+                            >
+                                <MenuItem
+                                    onClick={() => {
+                                        if (canEditProps) {
+                                            setSelectedProperty(record);
+                                            setShowConfirmDelete(true);
+                                        }
+                                    }}
+                                >
+                                    <Text color="red">Delete </Text>
+                                </MenuItem>
+                            </Tooltip>
+                        ),
+                    },
+                ];
+                return (
+                    <>
+                        <CardIcons>
+                            <Dropdown menu={{ items }} trigger={['click']}>
+                                <Icon icon="MoreVert" size="md" data-testid="structured-props-more-options-icon" />
+                            </Dropdown>
+                        </CardIcons>
+                    </>
+                );
+            },
+        },
+    ];
+    return (
+        <>
+            <Table
+                columns={columns}
+                data={filteredProperties}
+                isLoading={loading}
+                isScrollable
+                data-testid="structured-props-table"
+            />
+            <ConfirmationModal
+                isOpen={showConfirmDelete}
+                handleClose={handleDeleteClose}
+                handleConfirm={() => handleDeleteProperty(selectedProperty)}
+                modalTitle="Confirm Delete"
+                modalText="Are you sure you want to delete? Deleting will remove this structured property from all assets it's currently on."
+            />
+        </>
+    );
+};
+
+export default StructuredPropsTable;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx b/datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx
new file mode 100644
index 0000000000000..25f1d67239042
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx
@@ -0,0 +1,43 @@
+import { Icon, Text } from '@components';
+import { StructuredPropertyEntity } from '@src/types.generated';
+import { Collapse } from 'antd';
+import React from 'react';
+import { CollapseHeader, RowContainer, StyledCollapse, StyledLabel } from './styledComponents';
+
+interface Props {
+    propEntity: StructuredPropertyEntity;
+}
+
+const ViewAdvancedOptions = ({ propEntity }: Props) => {
+    return (
+        <StyledCollapse
+            ghost
+            expandIcon={({ isActive }) => (
+                <Icon icon="ChevronRight" color="gray" size="4xl" rotate={isActive ? '90' : '0'} />
+            )}
+            expandIconPosition="end"
+            defaultActiveKey={[]}
+        >
+            <Collapse.Panel
+                key={1}
+                header={
+                    <CollapseHeader>
+                        <Text weight="bold" color="gray" size="lg">
+                            Advanced Options
+                        </Text>
+                    </CollapseHeader>
+                }
+                forceRender
+            >
+                {propEntity && (
+                    <RowContainer>
+                        <StyledLabel>Qualified Name</StyledLabel>
+                        <Text color="gray"> {propEntity?.definition?.qualifiedName}</Text>
+                    </RowContainer>
+                )}
+            </Collapse.Panel>
+        </StyledCollapse>
+    );
+};
+
+export default ViewAdvancedOptions;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/ViewDisplayPreferences.tsx b/datahub-web-react/src/app/govern/structuredProperties/ViewDisplayPreferences.tsx
new file mode 100644
index 0000000000000..a1baeaeb618f7
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/ViewDisplayPreferences.tsx
@@ -0,0 +1,86 @@
+import { colors, Icon, Switch, Text } from '@src/alchemy-components';
+import { StructuredPropertyEntity } from '@src/types.generated';
+import { Collapse } from 'antd';
+import React from 'react';
+import { CollapseHeader, StyledCollapse, StyledFormItem, TogglesContainer } from './styledComponents';
+
+interface Props {
+    propEntity: StructuredPropertyEntity;
+}
+
+const ViewDisplayPreferences = ({ propEntity }: Props) => {
+    return (
+        <>
+            <StyledCollapse
+                ghost
+                expandIcon={({ isActive }) => (
+                    <Icon icon="ChevronRight" color="gray" size="4xl" rotate={isActive ? '90' : '0'} />
+                )}
+                expandIconPosition="end"
+                defaultActiveKey={[1]}
+            >
+                <Collapse.Panel
+                    key={1}
+                    header={
+                        <CollapseHeader>
+                            <Text weight="bold" color="gray">
+                                Display Preferences
+                            </Text>
+                        </CollapseHeader>
+                    }
+                    forceRender
+                >
+                    <TogglesContainer>
+                        <StyledFormItem name={['settings', 'isHidden']}>
+                            <Switch
+                                label="Hide Property"
+                                size="sm"
+                                checked={propEntity?.settings?.isHidden}
+                                labelStyle={{ fontSize: 12, color: colors.gray[1700], fontWeight: 700 }}
+                                isDisabled
+                            />
+                        </StyledFormItem>
+                        <StyledFormItem name={['settings', 'showInSearchFilters']}>
+                            <Switch
+                                label="Show in Search Filters"
+                                size="sm"
+                                checked={propEntity?.settings?.showInSearchFilters}
+                                labelStyle={{ fontSize: 12, color: colors.gray[1700], fontWeight: 700 }}
+                                isDisabled
+                            />
+                        </StyledFormItem>
+                        <StyledFormItem name={['settings', 'showInAssetSummary']}>
+                            <Switch
+                                label="Show in Asset Sidebar"
+                                size="sm"
+                                checked={propEntity?.settings?.showInAssetSummary}
+                                labelStyle={{ fontSize: 12, color: colors.gray[1700], fontWeight: 700 }}
+                                isDisabled
+                            />
+                        </StyledFormItem>
+                        <StyledFormItem name={['settings', 'showAsAssetBadge']}>
+                            <Switch
+                                label="Show as Asset Badge"
+                                size="sm"
+                                checked={propEntity?.settings?.showAsAssetBadge === true}
+                                labelStyle={{ fontSize: 12, color: colors.gray[1700], fontWeight: 700 }}
+                                isDisabled
+                            />
+                        </StyledFormItem>
+                        <StyledFormItem name={['settings', 'showInColumnsTable']}>
+                            <Switch
+                                label="Show in Columns Table"
+                                size="sm"
+                                checked={propEntity?.settings?.showInColumnsTable}
+                                labelStyle={{ fontSize: 12, color: colors.gray[1700], fontWeight: 700 }}
+                                isDisabled
+                            />
+                        </StyledFormItem>
+                    </TogglesContainer>
+                </Collapse.Panel>
+            </StyledCollapse>
+        </>
+    );
+};
+
+export default ViewDisplayPreferences;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx b/datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx
new file mode 100644
index 0000000000000..bc91a90989d2c
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx
@@ -0,0 +1,142 @@
+import { Text } from '@components';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { PropertyCardinality, SearchResult, StructuredPropertyEntity } from '@src/types.generated';
+import React from 'react';
+import {
+    DescriptionContainer,
+    DrawerHeader,
+    ItemsList,
+    RowContainer,
+    StyledDrawer,
+    StyledIcon,
+    StyledLabel,
+    VerticalDivider,
+    ViewDivider,
+    ViewFieldsContainer,
+} from './styledComponents';
+import { getDisplayName, getValueTypeLabel } from './utils';
+import ViewAdvancedOptions from './ViewAdvancedOptions';
+import ViewDisplayPreferences from './ViewDisplayPreferences';
+
+interface Props {
+    isViewDrawerOpen: boolean;
+    setIsViewDrawerOpen: React.Dispatch<React.SetStateAction<boolean>>;
+    selectedProperty: SearchResult;
+    setSelectedProperty: React.Dispatch<React.SetStateAction<SearchResult | undefined>>;
+}
+
+const ViewStructuredPropsDrawer = ({
+    isViewDrawerOpen,
+    setIsViewDrawerOpen,
+    selectedProperty,
+    setSelectedProperty,
+}: Props) => {
+    const entityRegistry = useEntityRegistry();
+
+    const handleClose = () => {
+        setIsViewDrawerOpen(false);
+        setSelectedProperty(undefined);
+    };
+
+    const selectedPropEntity = selectedProperty && (selectedProperty?.entity as StructuredPropertyEntity);
+
+    const allowedValues = selectedPropEntity?.definition?.allowedValues;
+
+    const allowedTypes = selectedPropEntity?.definition?.typeQualifier?.allowedTypes;
+
+    const propType = getValueTypeLabel(
+        selectedPropEntity.definition.valueType.urn,
+        selectedPropEntity.definition.cardinality || PropertyCardinality.Single,
+    );
+
+    return (
+        <StyledDrawer
+            open={isViewDrawerOpen}
+            closable={false}
+            width={480}
+            title={
+                <>
+                    {selectedProperty && (
+                        <DrawerHeader>
+                            <Text color="gray" weight="bold" size="2xl">
+                                {getDisplayName(selectedProperty?.entity as StructuredPropertyEntity)}
+                            </Text>
+
+                            <StyledIcon icon="Close" color="gray" onClick={handleClose} />
+                        </DrawerHeader>
+                    )}
+                </>
+            }
+            footer={null}
+            destroyOnClose
+        >
+            <ViewFieldsContainer>
+                {selectedPropEntity.definition.description && (
+                    <DescriptionContainer>
+                        <Text weight="bold" color="gray" size="lg">
+                            Description
+                        </Text>
+                        <Text color="gray"> {selectedPropEntity.definition.description}</Text>
+                        <ViewDivider />
+                    </DescriptionContainer>
+                )}
+                <RowContainer>
+                    <StyledLabel>Property Type</StyledLabel>
+                    <Text color="gray"> {propType}</Text>
+                </RowContainer>
+                {allowedTypes && allowedTypes.length > 0 && (
+                    <RowContainer>
+                        <StyledLabel>Allowed Entity Types</StyledLabel>
+                        <ItemsList>
+                            {allowedTypes.map((type, index) => {
+                                return (
+                                    <>
+                                        <Text color="gray">{entityRegistry.getEntityName(type.info.type)}</Text>
+                                        {index < allowedTypes.length - 1 && <VerticalDivider type="vertical" />}
+                                    </>
+                                );
+                            })}
+                        </ItemsList>
+                    </RowContainer>
+                )}
+                {allowedValues && allowedValues.length > 0 && (
+                    <RowContainer>
+                        <StyledLabel>Allowed Values</StyledLabel>
+                        <ItemsList>
+                            {allowedValues?.map((val, index) => {
+                                return (
+                                    <>
+                                        <Text color="gray">
+                                            {(val.value as any).stringValue || (val.value as any).numberValue}
+                                        </Text>
+                                        {index < allowedValues?.length - 1 && <VerticalDivider type="vertical" />}
+                                    </>
+                                );
+                            })}
+                        </ItemsList>
+                    </RowContainer>
+                )}
+
+                <RowContainer>
+                    <StyledLabel>Applies To</StyledLabel>
+                    <ItemsList>
+                        {selectedPropEntity.definition.entityTypes?.map((type, index) => {
+                            return (
+                                <>
+                                    <Text color="gray">{entityRegistry.getEntityName(type.info.type)}</Text>
+                                    {index < selectedPropEntity.definition.entityTypes?.length - 1 && (
+                                        <VerticalDivider type="vertical" />
+                                    )}
+                                </>
+                            );
+                        })}
+                    </ItemsList>
+                </RowContainer>
+            </ViewFieldsContainer>
+            <ViewDisplayPreferences propEntity={selectedPropEntity} />
+            <ViewAdvancedOptions propEntity={selectedPropEntity} />
+        </StyledDrawer>
+    );
+};
+
+export default ViewStructuredPropsDrawer;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/cacheUtils.ts b/datahub-web-react/src/app/govern/structuredProperties/cacheUtils.ts
new file mode 100644
index 0000000000000..c8052784c6972
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/cacheUtils.ts
@@ -0,0 +1,101 @@
+import { GetSearchResultsForMultipleDocument, GetSearchResultsForMultipleQuery } from '@src/graphql/search.generated';
+
+const addToCache = (existingProperties, newProperty) => {
+    const propertyToWrite = {
+        entity: {
+            urn: newProperty.urn,
+            type: newProperty.type,
+            definition: {
+                displayName: newProperty.definition.displayName,
+                qualifiedName: newProperty.definition.qualifiedName,
+                description: newProperty.definition.description,
+                cardinality: newProperty.definition.cardinality,
+                immutable: newProperty.definition.immutable,
+                valueType: newProperty.definition.valueType,
+                entityTypes: newProperty.definition.entityTypes,
+                typeQualifier: newProperty.definition.typeQualifier,
+                allowedValues: newProperty.definition.allowedValues,
+                created: newProperty.definition.created,
+                lastModified: newProperty.definition.lastModified,
+            },
+            settings: {
+                isHidden: newProperty.settings.isHidden,
+                showInSearchFilters: newProperty.settings.showInSearchFilters,
+                showAsAssetBadge: newProperty.settings.showAsAssetBadge,
+                showInAssetSummary: newProperty.settings.showInAssetSummary,
+                showInColumnsTable: newProperty.settings.showInColumnsTable,
+            },
+            __typename: 'StructuredPropertyEntity',
+        },
+        matchedFields: [],
+        insights: [],
+        extraProperties: [],
+        __typename: 'SearchResult',
+    };
+
+    return [propertyToWrite, ...existingProperties];
+};
+
+export const updatePropertiesList = (client, inputs, newProperty, searchAcrossEntities) => {
+    //  Read the data from our cache for this query.
+    const currData: GetSearchResultsForMultipleQuery | null = client.readQuery({
+        query: GetSearchResultsForMultipleDocument,
+        variables: {
+            input: inputs,
+        },
+    });
+
+    if (currData === null) {
+        // If there's no cached data, the first load has not occurred. Let it occur naturally.
+        return;
+    }
+
+    const existingProperties = currData?.searchAcrossEntities?.searchResults || [];
+    const newProperties = addToCache(existingProperties, newProperty);
+
+    // Write our data back to the cache.
+    client.writeQuery({
+        query: GetSearchResultsForMultipleDocument,
+        variables: {
+            input: inputs,
+        },
+        data: {
+            searchAcrossEntities: {
+                ...searchAcrossEntities,
+                total: newProperties.length,
+                searchResults: newProperties,
+            },
+        },
+    });
+};
+
+export const removeFromPropertiesList = (client, inputs, deleteUrn, searchAcrossEntities) => {
+    const currData: GetSearchResultsForMultipleQuery | null = client.readQuery({
+        query: GetSearchResultsForMultipleDocument,
+        variables: {
+            input: inputs,
+        },
+    });
+
+    if (currData === null) {
+        return;
+    }
+
+    const existingProperties = currData?.searchAcrossEntities?.searchResults || [];
+
+    const newProperties = [...existingProperties.filter((prop) => prop.entity.urn !== deleteUrn)];
+
+    client.writeQuery({
+        query: GetSearchResultsForMultipleDocument,
+        variables: {
+            input: inputs,
+        },
+        data: {
+            searchAcrossEntities: {
+                ...searchAcrossEntities,
+                total: newProperties.length,
+                searchResults: newProperties,
+            },
+        },
+    });
+};
diff --git a/datahub-web-react/src/app/govern/structuredProperties/styledComponents.ts b/datahub-web-react/src/app/govern/structuredProperties/styledComponents.ts
new file mode 100644
index 0000000000000..211b4cdab6638
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/styledComponents.ts
@@ -0,0 +1,473 @@
+import { colors, Icon, typography } from '@src/alchemy-components';
+import { Checkbox, Collapse, Divider, Drawer, Form, Modal, Select, Spin, Typography } from 'antd';
+import styled from 'styled-components';
+
+export const PageContainer = styled.div<{ $isShowNavBarRedesign?: boolean }>`
+    overflow: auto;
+    margin: ${(props) => (props.$isShowNavBarRedesign ? '0' : '0 12px 12px 0')};
+    padding: 16px 20px 20px 20px;
+    border-radius: ${(props) =>
+        props.$isShowNavBarRedesign ? props.theme.styles['border-radius-navbar-redesign'] : '8px'};
+    display: flex;
+    flex: 1;
+    flex-direction: column;
+    gap: 20px;
+    background-color: ${colors.white};
+    ${(props) => props.$isShowNavBarRedesign && 'max-height: calc(100vh - 88px);'};
+    ${(props) =>
+        props.$isShowNavBarRedesign &&
+        `
+        box-shadow: ${props.theme.styles['box-shadow-navbar-redesign']};
+        margin: 5px;
+    `}
+`;
+
+export const HeaderContainer = styled.div`
+    display: flex;
+    justify-content: space-between;
+`;
+
+export const TableContainer = styled.div`
+    display: flex;
+    overflow: auto;
+    flex: 1;
+`;
+
+export const HeaderContent = styled.div`
+    display: flex;
+    flex-direction: column;
+`;
+
+export const ButtonContainer = styled.div`
+    display: flex;
+    align-self: center;
+`;
+
+export const DataContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    width: calc(100% - 44px);
+`;
+
+export const PropName = styled(Typography.Text)`
+    font-size: 14px;
+    font-weight: 600;
+    color: ${colors.gray[600]};
+    line-height: normal;
+
+    :hover {
+        cursor: pointer;
+        text-decoration: underline;
+    }
+`;
+
+export const PropDescription = styled(Typography.Text)`
+    font-size: 14px;
+    font-weight: 400;
+    color: ${colors.gray[1700]};
+    line-height: normal;
+`;
+
+export const NameColumn = styled.div`
+    display: flex;
+    gap: 12px;
+    align-items: center;
+`;
+
+export const IconContainer = styled.div`
+    height: 32px;
+    width: 32px;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    gap: 12px;
+    border-radius: 200px;
+    background-color: ${colors.gray[1000]};
+`;
+
+export const PillsContainer = styled.div`
+    display: flex;
+    align-items: center;
+    gap: 8px;
+`;
+
+export const PillContainer = styled.div`
+    display: flex;
+`;
+
+export const MenuItem = styled.div`
+    display: flex;
+    padding: 5px 100px 5px 5px;
+    font-size: 14px;
+    font-weight: 400;
+    color: ${colors.gray[600]};
+    font-family: ${typography.fonts.body};
+`;
+
+export const DrawerHeader = styled.div`
+    display: flex;
+    justify-content: space-between;
+`;
+
+export const TitleContainer = styled.div`
+    display: flex;
+    align-items: center;
+    gap: 6px;
+`;
+
+export const StyledIcon = styled(Icon)`
+    &:hover {
+        cursor: pointer;
+    }
+`;
+
+export const FooterContainer = styled.div`
+    width: 100%;
+`;
+
+export const RowContainer = styled.div`
+    display: grid;
+    grid-template-columns: 180px 1fr;
+    align-items: center;
+`;
+
+export const ViewFieldsContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    gap: 24px;
+`;
+
+export const CheckboxWrapper = styled.div`
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    margin: 10px 0;
+    color: #374066;
+    p {
+        color: #374066;
+        font-weight: 500;
+    }
+`;
+
+export const StyledCheckbox = styled(Checkbox)`
+    .ant-checkbox-checked .ant-checkbox-inner {
+        background-color: ${colors.violet[500]};
+        border-color: ${colors.violet[500]} !important;
+    },
+`;
+
+export const StyledText = styled.div`
+    display: inline-flex;
+    margin-left: -4px;
+`;
+
+export const StyledFormItem = styled(Form.Item)`
+    margin: 0;
+`;
+
+export const GridFormItem = styled(Form.Item)`
+    display: grid;
+`;
+
+export const FieldLabel = styled.div`
+    font-size: 14px;
+    font-weight: 500;
+    color: ${colors.gray[600]};
+    margin-bottom: 24px;
+`;
+
+export const InputLabel = styled.div`
+    font-size: 14px;
+    font-weight: 500;
+    color: ${colors.gray[600]};
+    display: flex;
+    gap: 2px;
+`;
+
+export const StyledLabel = styled.div`
+    font-size: 12px;
+    font-weight: 700;
+    color: ${colors.gray[1700]};
+`;
+
+export const DescriptionContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+`;
+
+export const FlexContainer = styled.div`
+    display: flex;
+    align-items: center;
+    gap: 4px;
+`;
+
+export const StyledDrawer = styled(Drawer)`
+    .ant-drawer-body {
+        padding: 16px;
+    }
+
+    .ant-drawer-header {
+        padding: 16px;
+    }
+`;
+
+export const StyledCollapse = styled(Collapse)`
+    .ant-collapse-header {
+        padding: 0 !important;
+    }
+
+    .ant-collapse-content-box {
+        padding: 12px 0 !important;
+    }
+
+    .ant-collapse-arrow {
+        right: 0 !important;
+    }
+`;
+
+export const CollapseHeader = styled.div`
+    border-top: 1px solid ${colors.gray[1400]};
+    padding: 16px 0;
+    margin-left: -16px;
+    width: calc(100% + 32px);
+    padding: 16px;
+    margin-top: 12px;
+`;
+
+export const TogglesContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    gap: 32px;
+`;
+
+export const StyledSpin = styled(Spin)`
+    max-height: 100% !important;
+    color: ${colors.violet[500]};
+`;
+
+export const CreatedByContainer = styled.div`
+    display: inline-flex;
+    align-items: center;
+    gap: 4px;
+    padding: 3px 6px 3px 4px;
+    border-radius: 20px;
+    border: 1px solid ${colors.gray[1400]};
+
+    :hover {
+        cursor: pointer;
+    }
+`;
+
+export const SubTextContainer = styled.div`
+    display: flex;
+    align-items: center;
+    gap: 4px;
+
+    p {
+        margin: 0;
+    }
+`;
+
+export const ValueListContainer = styled.div`
+    margin-bottom: 24px;
+    display: flex;
+    align-items: center;
+    gap: 4px;
+    font-size: 14px;
+    color: ${colors.gray[500]};
+
+    svg {
+        :hover {
+            cursor: pointer;
+        }
+    }
+`;
+
+export const ValueType = styled.div`
+    background-color: ${colors.gray[100]};
+    border-radius: 4px;
+    padding: 2px 4px;
+`;
+
+export const StyledModal = styled(Modal)`
+    font-family: Mulish;
+
+    &&& .ant-modal-content {
+        box-shadow: 0px 4px 12px 0px rgba(9, 1, 61, 0.12);
+        border-radius: 12px;
+        width: 452px;
+    }
+
+    .ant-modal-header {
+        border-top-left-radius: 12px !important;
+        border-top-right-radius: 12px !important;
+        padding: 20px 20px 8px 20px;
+    }
+
+    .ant-modal-body {
+        padding: 0;
+    }
+`;
+
+export const FieldGroupContainer = styled.div`
+    display: grid;
+    margin-bottom: 8px;
+`;
+
+export const DeleteIconContainer = styled.div`
+    display: flex;
+    margin-top: -16px;
+    justify-self: end;
+
+    :hover {
+        cursor: pointer;
+    }
+`;
+
+export const ValuesContainer = styled.div<{ height: number }>`
+    max-height: ${(props) => `calc(${props.height}px - 200px)`};
+    overflow: auto;
+    padding: 20px;
+`;
+
+export const ValuesList = styled.div`
+    font-size: 14px;
+    color: ${colors.gray[500]};
+    display: flex;
+    flex: 1;
+    align-items: center;
+    flex-wrap: wrap;
+
+    p {
+        line-height: 24px;
+    }
+`;
+
+export const ItemsList = styled.div`
+    font-size: 14px;
+    display: flex;
+    align-items: center;
+    flex-wrap: wrap;
+`;
+
+export const ItemsContainer = styled.div`
+    display: flex;
+    align-items: center;
+    gap: 16px;
+    width: fit-content;
+    margin: 0 24px 24px 0;
+`;
+
+export const AddButtonContainer = styled.div`
+    display: flex;
+    margin: 10px 20px 0 0;
+    justify-self: end;
+`;
+
+export const FormContainer = styled.div`
+    display: grid;
+`;
+
+export const ModalFooter = styled.div`
+    display: flex;
+    gap: 16px;
+    justify-content: end;
+`;
+
+export const VerticalDivider = styled(Divider)`
+    color: ${colors.gray[1400]};
+    height: 20px;
+    width: 2px;
+`;
+
+export const StyledSelect = styled(Select)`
+    font-family: ${typography.fonts.body};
+
+    .ant-select-selector {
+        height: 50px !important;
+        border-radius: 8px !important;
+        outline: 2px solid transparent !important;
+
+        &:hover,
+        &:focus-within {
+            border: 1px solid ${colors.violet[200]} !important;
+            outline: 2px solid ${colors.violet[200]} !important;
+            box-shadow: none !important;
+        }
+    }
+
+    .ant-select-selection-item {
+        p {
+            text-overflow: ellipsis;
+            white-space: nowrap;
+            overflow: hidden;
+        }
+    }
+
+    .ant-select-selection-placeholder {
+        display: flex;
+        align-items: center;
+        font-size: 14px;
+        color: ${colors.gray[400]};
+    }
+`;
+
+export const CustomDropdown = styled.div`
+    .ant-select-item-option-content {
+        white-space: normal;
+    }
+`;
+
+export const SelectOptionContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    padding: 4px 0;
+
+    p {
+        line-height: 20px;
+    }
+`;
+
+export const StyledDivider = styled(Divider)`
+    color: ${colors.gray[1400]};
+    margin: 16px 0;
+`;
+
+export const ViewDivider = styled(Divider)`
+    color: ${colors.gray[1400]};
+    margin: 16px 0 0 -16px;
+    width: calc(100% + 32px);
+`;
+
+export const EmptyContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    height: 100%;
+    width: 100%;
+    gap: 16px;
+
+    svg {
+        width: 160px;
+        height: 160px;
+    }
+`;
+
+export const CardIcons = styled.div`
+    display: flex;
+    justify-content: end;
+    gap: 12px;
+
+    div {
+        border: 1px solid $E9EAEE;
+        border-radius: 20px;
+        width: 28px;
+        height: 28px;
+        padding: 4px;
+        color: #8088a3;
+        :hover {
+            cursor: pointer;
+        }
+    }
+`;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/useStructuredProp.ts b/datahub-web-react/src/app/govern/structuredProperties/useStructuredProp.ts
new file mode 100644
index 0000000000000..6de195c1cd424
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/useStructuredProp.ts
@@ -0,0 +1,135 @@
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
+import { EntityType, PropertyCardinality, SearchResult, StructuredPropertyEntity } from '@src/types.generated';
+import { FormInstance } from 'antd';
+import { useMemo } from 'react';
+import { getEntityTypeUrn, StructuredProp, valueTypes } from './utils';
+
+interface Props {
+    selectedProperty?: SearchResult;
+    form: FormInstance;
+    setFormValues: React.Dispatch<React.SetStateAction<StructuredProp | undefined>>;
+    setCardinality: React.Dispatch<React.SetStateAction<PropertyCardinality>>;
+    setSelectedValueType: React.Dispatch<React.SetStateAction<string>>;
+}
+
+export default function useStructuredProp({
+    selectedProperty,
+    form,
+    setFormValues,
+    setCardinality,
+    setSelectedValueType,
+}: Props) {
+    const entityRegistry = useEntityRegistry();
+
+    const getEntitiesListOptions = (entitiesList: EntityType[]) => {
+        const listOptions: { label: string; value: string }[] = [];
+        entitiesList.forEach((type) => {
+            const entity = {
+                label: entityRegistry.getEntityName(type) || '',
+                value: getEntityTypeUrn(entityRegistry, type),
+            };
+            listOptions.push(entity);
+        });
+        return listOptions;
+    };
+
+    const updateFormValues = (field, values) => {
+        if (field.includes('typeQualifier')) {
+            setFormValues((prev) => ({
+                ...prev,
+                typeQualifier: {
+                    allowedTypes: values,
+                },
+            }));
+        } else
+            setFormValues((prev) => ({
+                ...prev,
+                [field]: values,
+            }));
+    };
+
+    const handleSelectChange = (field, values) => {
+        form.setFieldValue(field, values);
+        updateFormValues(field, values);
+    };
+
+    const handleSelectUpdateChange = (field, values) => {
+        const entity = selectedProperty?.entity as StructuredPropertyEntity;
+        let initialValues: string[] = [];
+
+        if (field === 'entityTypes') initialValues = entity.definition.entityTypes.map((type) => type.urn);
+
+        if (field.includes('typeQualifier'))
+            initialValues = entity.definition.typeQualifier?.allowedTypes?.map((type) => type.urn) || [];
+
+        const updatedValues = [...initialValues, ...values.filter((value) => !initialValues.includes(value))];
+
+        form.setFieldValue(field, updatedValues);
+        updateFormValues(field, updatedValues);
+    };
+
+    // Handle change in the property type dropdown
+    const handleTypeUpdate = (value: string) => {
+        const typeOption = valueTypes.find((type) => type.value === value);
+        setSelectedValueType(value);
+        handleSelectChange('valueType', value);
+        setFormValues((prev) => ({
+            ...prev,
+            valueType: value,
+        }));
+
+        const isList = typeOption?.cardinality === PropertyCardinality.Multiple;
+        if (isList) setCardinality(PropertyCardinality.Multiple);
+        else setCardinality(PropertyCardinality.Single);
+    };
+
+    const settingsDefault = {
+        isHidden: false,
+        showInSearchFilters: false,
+        showAsAssetBadge: false,
+        showInAssetSummary: false,
+        showInColumnsTable: false,
+    };
+
+    const handleDisplaySettingChange = (settingField: string, value: boolean) => {
+        if (settingField === 'isHidden' && value) {
+            Object.keys(settingsDefault).forEach((settingKey) => form.setFieldValue(['settings', settingKey], false));
+            setFormValues((prev) => ({
+                ...prev,
+                settings: {
+                    ...settingsDefault,
+                    [settingField]: value,
+                },
+            }));
+        } else {
+            setFormValues((prev) => ({
+                ...prev,
+                settings: {
+                    ...(prev?.settings || settingsDefault),
+                    [settingField]: value,
+                },
+            }));
+        }
+        form.setFieldValue(['settings', settingField], value);
+    };
+
+    const disabledEntityTypeValues = useMemo(() => {
+        return (selectedProperty?.entity as StructuredPropertyEntity)?.definition?.entityTypes?.map((type) => type.urn);
+    }, [selectedProperty]);
+
+    const disabledTypeQualifierValues = useMemo(() => {
+        return (selectedProperty?.entity as StructuredPropertyEntity)?.definition?.typeQualifier?.allowedTypes?.map(
+            (type) => type.urn,
+        );
+    }, [selectedProperty]);
+
+    return {
+        handleSelectChange,
+        handleSelectUpdateChange,
+        handleTypeUpdate,
+        getEntitiesListOptions,
+        disabledEntityTypeValues,
+        disabledTypeQualifierValues,
+        handleDisplaySettingChange,
+    };
+}
diff --git a/datahub-web-react/src/app/govern/structuredProperties/utils.ts b/datahub-web-react/src/app/govern/structuredProperties/utils.ts
new file mode 100644
index 0000000000000..744ce20cf4573
--- /dev/null
+++ b/datahub-web-react/src/app/govern/structuredProperties/utils.ts
@@ -0,0 +1,249 @@
+import EntityRegistry from '@src/app/entity/EntityRegistry';
+import { mapStructuredPropertyToPropertyRow } from '@src/app/entity/shared/tabs/Properties/useStructuredProperties';
+import {
+    ENTITY_TYPES_FILTER_NAME,
+    IS_HIDDEN_PROPERTY_FILTER_NAME,
+    SHOW_IN_ASSET_SUMMARY_PROPERTY_FILTER_NAME,
+    SHOW_IN_COLUMNS_TABLE_PROPERTY_FILTER_NAME,
+} from '@src/app/search/utils/constants';
+import {
+    AllowedValue,
+    EntityType,
+    FacetFilterInput,
+    Maybe,
+    PropertyCardinality,
+    SearchResult,
+    StructuredProperties,
+    StructuredPropertyEntity,
+    StructuredPropertySettings,
+} from '@src/types.generated';
+
+export type StructuredProp = {
+    displayName?: string;
+    qualifiedName?: string;
+    cardinality?: PropertyCardinality;
+    description?: string | null;
+    valueType?: string;
+    entityTypes?: string[];
+    typeQualifier?: {
+        allowedTypes?: string[];
+    };
+    immutable?: boolean;
+    allowedValues?: AllowedValue[];
+    settings?: StructuredPropertySettings | null;
+};
+
+export const valueTypes = [
+    {
+        urn: 'urn:li:dataType:datahub.string',
+        label: 'Text',
+        value: 'string',
+        cardinality: PropertyCardinality.Single,
+        description: 'A string value',
+    },
+    {
+        urn: 'urn:li:dataType:datahub.string',
+        label: 'Text - List',
+        value: 'stringList',
+        cardinality: PropertyCardinality.Multiple,
+        description: 'A list of string values',
+    },
+    {
+        urn: 'urn:li:dataType:datahub.number',
+        label: 'Number',
+        value: 'number',
+        cardinality: PropertyCardinality.Single,
+        description: 'An integer or decimal',
+    },
+    {
+        urn: 'urn:li:dataType:datahub.number',
+        label: 'Number - List',
+        value: 'numberList',
+        cardinality: PropertyCardinality.Multiple,
+        description: 'A list of integers or decimals',
+    },
+    {
+        urn: 'urn:li:dataType:datahub.urn',
+        label: 'Entity',
+        value: 'entity',
+        cardinality: PropertyCardinality.Single,
+        description: 'A reference to a DataHub asset',
+    },
+    {
+        urn: 'urn:li:dataType:datahub.urn',
+        label: 'Entity - List',
+        value: 'entityList',
+        cardinality: PropertyCardinality.Multiple,
+        description: 'A reference to a list of DataHub assets',
+    },
+    {
+        urn: 'urn:li:dataType:datahub.rich_text',
+        label: 'Rich Text',
+        value: 'richText',
+        cardinality: PropertyCardinality.Single,
+        description: 'A freeform string of markdown text ',
+    },
+    {
+        urn: 'urn:li:dataType:datahub.date',
+        label: 'Date',
+        value: 'date',
+        cardinality: PropertyCardinality.Single,
+        description: 'A specific date',
+    },
+];
+
+export const SEARCHABLE_ENTITY_TYPES = [
+    EntityType.Dataset,
+    EntityType.DataJob,
+    EntityType.DataFlow,
+    EntityType.Chart,
+    EntityType.Dashboard,
+    EntityType.Domain,
+    EntityType.Container,
+    EntityType.GlossaryTerm,
+    EntityType.GlossaryNode,
+    EntityType.Mlmodel,
+    EntityType.MlmodelGroup,
+    EntityType.Mlfeature,
+    EntityType.MlfeatureTable,
+    EntityType.MlprimaryKey,
+    EntityType.DataProduct,
+    EntityType.CorpUser,
+    EntityType.CorpGroup,
+    EntityType.Tag,
+    EntityType.Role,
+];
+
+export const APPLIES_TO_ENTITIES = [
+    EntityType.Dataset,
+    EntityType.DataJob,
+    EntityType.DataFlow,
+    EntityType.Chart,
+    EntityType.Dashboard,
+    EntityType.Domain,
+    EntityType.Container,
+    EntityType.GlossaryTerm,
+    EntityType.GlossaryNode,
+    EntityType.Mlmodel,
+    EntityType.MlmodelGroup,
+    EntityType.Mlfeature,
+    EntityType.MlfeatureTable,
+    EntityType.MlprimaryKey,
+    EntityType.DataProduct,
+    EntityType.SchemaField,
+];
+
+export const getEntityTypeUrn = (entityRegistry: EntityRegistry, entityType: EntityType) => {
+    return `urn:li:entityType:datahub.${entityRegistry.getGraphNameFromType(entityType)}`;
+};
+
+export function getDisplayName(structuredProperty: StructuredPropertyEntity) {
+    return structuredProperty.definition.displayName || structuredProperty.definition.qualifiedName;
+}
+
+export const getValueType = (valueUrn: string, cardinality: PropertyCardinality) => {
+    return valueTypes.find((valueType) => valueType.urn === valueUrn && valueType.cardinality === cardinality)?.value;
+};
+
+export const getValueTypeLabel = (valueUrn: string, cardinality: PropertyCardinality) => {
+    return valueTypes.find((valueType) => valueType.urn === valueUrn && valueType.cardinality === cardinality)?.label;
+};
+
+export const getNewAllowedTypes = (entity: StructuredPropertyEntity, values: StructuredProp) => {
+    const currentTypeUrns = entity.definition.typeQualifier?.allowedTypes?.map((type) => type.urn);
+    return values.typeQualifier?.allowedTypes?.filter((type) => !currentTypeUrns?.includes(type));
+};
+
+export const getNewEntityTypes = (entity: StructuredPropertyEntity, values: StructuredProp) => {
+    const currentTypeUrns = entity.definition.entityTypes?.map((type) => type.urn);
+    return values.entityTypes?.filter((type) => !currentTypeUrns.includes(type));
+};
+
+export const getNewAllowedValues = (entity: StructuredPropertyEntity, values: StructuredProp) => {
+    const currentAllowedValues = entity.definition.allowedValues?.map(
+        (val: any) => val.value.numberValue || val.value.stringValue,
+    );
+    return values.allowedValues?.filter(
+        (val: any) =>
+            !(currentAllowedValues?.includes(val.stringValue) || currentAllowedValues?.includes(val.numberValue)),
+    );
+};
+
+export const isEntityTypeSelected = (selectedType: string) => {
+    if (selectedType === 'entity' || selectedType === 'entityList') return true;
+    return false;
+};
+
+export const isStringOrNumberTypeSelected = (selectedType: string) => {
+    if (
+        selectedType === 'string' ||
+        selectedType === 'stringList' ||
+        selectedType === 'number' ||
+        selectedType === 'numberList'
+    )
+        return true;
+    return false;
+};
+
+export const canBeAssetBadge = (selectedType: string, allowedValues?: AllowedValue[]) => {
+    if (selectedType === 'string' || selectedType === 'number') {
+        return !!allowedValues?.length;
+    }
+    return false;
+};
+
+export type PropValueField = 'stringValue' | 'numberValue';
+
+export const getStringOrNumberValueField = (selectedType: string) => {
+    if (selectedType === 'number' || selectedType === 'numberList') return 'numberValue' as PropValueField;
+    return 'stringValue' as PropValueField;
+};
+
+export const getPropertyRowFromSearchResult = (
+    property: SearchResult,
+    structuredProperties: Maybe<StructuredProperties> | undefined,
+) => {
+    const entityProp = structuredProperties?.properties?.find(
+        (prop) => prop.structuredProperty.urn === property.entity.urn,
+    );
+    return entityProp ? mapStructuredPropertyToPropertyRow(entityProp) : undefined;
+};
+
+export const getNotHiddenPropertyFilter = () => {
+    const isHiddenFilter: FacetFilterInput = {
+        field: IS_HIDDEN_PROPERTY_FILTER_NAME,
+        values: ['true'],
+        negated: true,
+    };
+    return isHiddenFilter;
+};
+
+export const getShowInColumnsTablePropertyFilter = () => {
+    const columnsTableFilter: FacetFilterInput = {
+        field: SHOW_IN_COLUMNS_TABLE_PROPERTY_FILTER_NAME,
+        values: ['true'],
+    };
+    return columnsTableFilter;
+};
+
+export const getShowInAssetSummaryPropertyFilter = () => {
+    const assetSummaryFilter: FacetFilterInput = {
+        field: SHOW_IN_ASSET_SUMMARY_PROPERTY_FILTER_NAME,
+        values: ['true'],
+    };
+    return assetSummaryFilter;
+};
+
+export const getEntityTypesPropertyFilter = (
+    entityRegistry: EntityRegistry,
+    isSchemaField: boolean,
+    entityType?: EntityType,
+) => {
+    const type = isSchemaField ? EntityType.SchemaField : entityType;
+
+    const entityTypesFilter: FacetFilterInput = {
+        field: ENTITY_TYPES_FILTER_NAME,
+        values: [getEntityTypeUrn(entityRegistry, type || EntityType.SchemaField)],
+    };
+    return entityTypesFilter;
+};
diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx
index c881109f6e419..586e100ac46f0 100644
--- a/datahub-web-react/src/app/home/HomePageHeader.tsx
+++ b/datahub-web-react/src/app/home/HomePageHeader.tsx
@@ -221,7 +221,7 @@ export const HomePageHeader = () => {
     const searchResultsToShow = useMemo(() => {
         let result: string[] | undefined = [];
         if (searchResultsData) {
-            const entities = searchResultsData?.searchAcrossEntities?.searchResults.map((searchResult) => {
+            const entities = searchResultsData?.searchAcrossEntities?.searchResults?.map((searchResult) => {
                 return searchResult?.entity;
             });
 
diff --git a/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx b/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx
index 5085a20b9b2bb..48dd2ae78ffc4 100644
--- a/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx
+++ b/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx
@@ -52,7 +52,7 @@ export const ManageIngestionPage = () => {
      */
     const me = useUserContext();
     const { config, loaded } = useAppConfig();
-    const isIngestionEnabled = config?.managedIngestionConfig.enabled;
+    const isIngestionEnabled = config?.managedIngestionConfig?.enabled;
     const showIngestionTab = isIngestionEnabled && me && me.platformPrivileges?.manageIngestion;
     const showSecretsTab = isIngestionEnabled && me && me.platformPrivileges?.manageSecrets;
     const [selectedTab, setSelectedTab] = useState<TabType>(TabType.Sources);
diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
index d9ab4cdc499f5..edb01a5acb07e 100644
--- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
+++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -17,7 +17,7 @@ import TabToolbar from '../../entity/shared/components/styled/TabToolbar';
 import { IngestionSourceBuilderModal } from './builder/IngestionSourceBuilderModal';
 import { addToListIngestionSourcesCache, CLI_EXECUTOR_ID, removeFromListIngestionSourcesCache } from './utils';
 import { DEFAULT_EXECUTOR_ID, SourceBuilderState, StringMapEntryInput } from './builder/types';
-import { IngestionSource, UpdateIngestionSourceInput } from '../../../types.generated';
+import { IngestionSource, SortCriterion, SortOrder, UpdateIngestionSourceInput } from '../../../types.generated';
 import { SearchBar } from '../../search/SearchBar';
 import { useEntityRegistry } from '../../useEntityRegistry';
 import { ExecutionDetailsModal } from './executions/ExecutionRequestDetailsModal';
@@ -60,16 +60,6 @@ export enum IngestionSourceType {
     CLI,
 }
 
-export function shouldIncludeSource(source: any, sourceFilter: IngestionSourceType) {
-    if (sourceFilter === IngestionSourceType.CLI) {
-        return source.config.executorId === CLI_EXECUTOR_ID;
-    }
-    if (sourceFilter === IngestionSourceType.UI) {
-        return source.config.executorId !== CLI_EXECUTOR_ID;
-    }
-    return true;
-}
-
 const DEFAULT_PAGE_SIZE = 25;
 
 const removeExecutionsFromIngestionSource = (source) => {
@@ -105,6 +95,7 @@ export const IngestionSourceList = () => {
     // Set of removed urns used to account for eventual consistency
     const [removedUrns, setRemovedUrns] = useState<string[]>([]);
     const [sourceFilter, setSourceFilter] = useState(IngestionSourceType.ALL);
+    const [sort, setSort] = useState<SortCriterion>();
     const [hideSystemSources, setHideSystemSources] = useState(true);
 
     /**
@@ -115,7 +106,14 @@ export const IngestionSourceList = () => {
     // Ingestion Source Default Filters
     const filters = hideSystemSources
         ? [{ field: 'sourceType', values: [SYSTEM_INTERNAL_SOURCE_TYPE], negated: true }]
-        : undefined;
+        : [];
+    if (sourceFilter !== IngestionSourceType.ALL) {
+        filters.push({
+            field: 'sourceExecutorId',
+            values: [CLI_EXECUTOR_ID],
+            negated: sourceFilter !== IngestionSourceType.CLI,
+        });
+    }
 
     // Ingestion Source Queries
     const { loading, error, data, client, refetch } = useListIngestionSourcesQuery({
@@ -123,8 +121,9 @@ export const IngestionSourceList = () => {
             input: {
                 start,
                 count: pageSize,
-                query: (query?.length && query) || undefined,
-                filters,
+                query: query?.length ? query : undefined,
+                filters: filters.length ? filters : undefined,
+                sort,
             },
         },
         fetchPolicy: (query?.length || 0) > 0 ? 'no-cache' : 'cache-first',
@@ -138,9 +137,7 @@ export const IngestionSourceList = () => {
 
     const totalSources = data?.listIngestionSources?.total || 0;
     const sources = data?.listIngestionSources?.ingestionSources || [];
-    const filteredSources = sources.filter(
-        (source) => !removedUrns.includes(source.urn) && shouldIncludeSource(source, sourceFilter),
-    ) as IngestionSource[];
+    const filteredSources = sources.filter((source) => !removedUrns.includes(source.urn)) as IngestionSource[];
     const focusSource =
         (focusSourceUrn && filteredSources.find((source) => source.urn === focusSourceUrn)) || undefined;
 
@@ -156,7 +153,7 @@ export const IngestionSourceList = () => {
 
     function hasActiveExecution() {
         return !!filteredSources.find((source) =>
-            source.executions?.executionRequests.find((request) => isExecutionRequestActive(request)),
+            source.executions?.executionRequests?.find((request) => isExecutionRequestActive(request)),
         );
     }
     useRefreshIngestionData(onRefresh, hasActiveExecution);
@@ -196,7 +193,9 @@ export const IngestionSourceList = () => {
 
     const formatExtraArgs = (extraArgs): StringMapEntryInput[] => {
         if (extraArgs === null || extraArgs === undefined) return [];
-        return extraArgs.map((entry) => ({ key: entry.key, value: entry.value }));
+        return extraArgs
+            .filter((entry) => entry.value !== null && entry.value !== undefined && entry.value !== '')
+            .map((entry) => ({ key: entry.key, value: entry.value }));
     };
 
     const createOrUpdateIngestionSource = (
@@ -376,6 +375,17 @@ export const IngestionSourceList = () => {
         setFocusSourceUrn(undefined);
     };
 
+    const onChangeSort = (field, order) => {
+        setSort(
+            order
+                ? {
+                      sortOrder: order === 'ascend' ? SortOrder.Ascending : SortOrder.Descending,
+                      field,
+                  }
+                : undefined,
+        );
+    };
+
     return (
         <>
             {!data && loading && <Message type="loading" content="Loading ingestion sources..." />}
@@ -438,6 +448,7 @@ export const IngestionSourceList = () => {
                     onView={onView}
                     onDelete={onDelete}
                     onRefresh={onRefresh}
+                    onChangeSort={onChangeSort}
                 />
                 <SourcePaginationContainer>
                     <Pagination
@@ -457,7 +468,7 @@ export const IngestionSourceList = () => {
                 onSubmit={onSubmit}
                 onCancel={onCancel}
             />
-            {isViewingRecipe && <RecipeViewerModal recipe={focusSource?.config.recipe} onCancel={onCancel} />}
+            {isViewingRecipe && <RecipeViewerModal recipe={focusSource?.config?.recipe} onCancel={onCancel} />}
             {focusExecutionUrn && (
                 <ExecutionDetailsModal urn={focusExecutionUrn} open onClose={() => setFocusExecutionUrn(undefined)} />
             )}
diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx
index 00d04ed245edf..2d785d7164853 100644
--- a/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx
+++ b/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx
@@ -1,6 +1,7 @@
 import { Empty, Typography } from 'antd';
 import React from 'react';
 import styled from 'styled-components/macro';
+import { SorterResult } from 'antd/lib/table/interface';
 import { StyledTable } from '../../entity/shared/components/styled/StyledTable';
 import { ANTD_GRAY } from '../../entity/shared/constants';
 import { CLI_EXECUTOR_ID, getIngestionSourceStatus } from './utils';
@@ -31,6 +32,7 @@ interface Props {
     onView: (urn: string) => void;
     onDelete: (urn: string) => void;
     onRefresh: () => void;
+    onChangeSort: (field: string, order: SorterResult<any>['order']) => void;
 }
 
 function IngestionSourceTable({
@@ -42,6 +44,7 @@ function IngestionSourceTable({
     onView,
     onDelete,
     onRefresh,
+    onChangeSort,
 }: Props) {
     const tableColumns = [
         {
@@ -49,14 +52,14 @@ function IngestionSourceTable({
             dataIndex: 'type',
             key: 'type',
             render: (type: string, record: any) => <TypeColumn type={type} record={record} />,
-            sorter: (sourceA, sourceB) => sourceA.type.localeCompare(sourceB.type),
+            sorter: true,
         },
         {
             title: 'Name',
             dataIndex: 'name',
             key: 'name',
             render: (name: string) => name || '',
-            sorter: (sourceA, sourceB) => sourceA.name.localeCompare(sourceB.name),
+            sorter: true,
         },
         {
             title: 'Schedule',
@@ -69,14 +72,12 @@ function IngestionSourceTable({
             dataIndex: 'execCount',
             key: 'execCount',
             render: (execCount: any) => <Typography.Text>{execCount || '0'}</Typography.Text>,
-            sorter: (sourceA, sourceB) => sourceA.execCount - sourceB.execCount,
         },
         {
             title: 'Last Execution',
             dataIndex: 'lastExecTime',
             key: 'lastExecTime',
             render: LastExecutionColumn,
-            sorter: (sourceA, sourceB) => sourceA.lastExecTime - sourceB.lastExecTime,
         },
         {
             title: 'Last Status',
@@ -85,7 +86,6 @@ function IngestionSourceTable({
             render: (status: any, record) => (
                 <LastStatusColumn status={status} record={record} setFocusExecutionUrn={setFocusExecutionUrn} />
             ),
-            sorter: (sourceA, sourceB) => (sourceA.lastExecStatus || '').localeCompare(sourceB.lastExecStatus || ''),
         },
         {
             title: '',
@@ -114,22 +114,30 @@ function IngestionSourceTable({
         execCount: source.executions?.total || 0,
         lastExecUrn:
             source.executions &&
-            source.executions?.executionRequests.length > 0 &&
-            source.executions?.executionRequests[0].urn,
+            source.executions?.executionRequests?.length > 0 &&
+            source.executions?.executionRequests[0]?.urn,
         lastExecTime:
             source.executions &&
-            source.executions?.executionRequests.length > 0 &&
-            source.executions?.executionRequests[0].result?.startTimeMs,
+            source.executions?.executionRequests?.length > 0 &&
+            source.executions?.executionRequests[0]?.result?.startTimeMs,
         lastExecStatus:
             source.executions &&
-            source.executions?.executionRequests.length > 0 &&
-            getIngestionSourceStatus(source.executions?.executionRequests[0].result),
+            source.executions?.executionRequests?.length > 0 &&
+            getIngestionSourceStatus(source.executions?.executionRequests[0]?.result),
         cliIngestion: source.config?.executorId === CLI_EXECUTOR_ID,
     }));
 
+    const handleTableChange = (_: any, __: any, sorter: any) => {
+        const sorterTyped: SorterResult<any> = sorter;
+        const field = sorterTyped.field as string;
+        const { order } = sorterTyped;
+        onChangeSort(field, order);
+    };
+
     return (
         <StyledSourceTable
             columns={tableColumns}
+            onChange={handleTableChange}
             dataSource={tableData}
             rowKey="urn"
             rowClassName={(record, _) => (record.cliIngestion ? 'cliIngestion' : '')}
diff --git a/datahub-web-react/src/app/ingest/source/__tests__/IngestionSourceList.test.tsx b/datahub-web-react/src/app/ingest/source/__tests__/IngestionSourceList.test.tsx
deleted file mode 100644
index b057f8b24c980..0000000000000
--- a/datahub-web-react/src/app/ingest/source/__tests__/IngestionSourceList.test.tsx
+++ /dev/null
@@ -1,38 +0,0 @@
-import { IngestionSourceType, shouldIncludeSource } from '../IngestionSourceList';
-import { CLI_EXECUTOR_ID } from '../utils';
-
-describe('shouldIncludeSource', () => {
-    it('should return true if the source filter is for CLI and the source is a CLI source', () => {
-        const source = { config: { executorId: CLI_EXECUTOR_ID } };
-        const isSourceIncluded = shouldIncludeSource(source, IngestionSourceType.CLI);
-        expect(isSourceIncluded).toBe(true);
-    });
-
-    it('should return false if the source filter is for CLI and the source is not a CLI source', () => {
-        const source = { config: { executorId: 'default' } };
-        const isSourceIncluded = shouldIncludeSource(source, IngestionSourceType.CLI);
-        expect(isSourceIncluded).toBe(false);
-    });
-
-    it('should return true if the source filter is for UI and the source is a UI source', () => {
-        const source = { config: { executorId: 'default' } };
-        const isSourceIncluded = shouldIncludeSource(source, IngestionSourceType.UI);
-        expect(isSourceIncluded).toBe(true);
-    });
-
-    it('should return false if the source filter is for UI and the source is not a UI source', () => {
-        const source = { config: { executorId: CLI_EXECUTOR_ID } };
-        const isSourceIncluded = shouldIncludeSource(source, IngestionSourceType.UI);
-        expect(isSourceIncluded).toBe(false);
-    });
-
-    it('should return true no matter what if the source type is all', () => {
-        const source1 = { config: { executorId: CLI_EXECUTOR_ID } };
-        const isSourceIncluded1 = shouldIncludeSource(source1, IngestionSourceType.ALL);
-        expect(isSourceIncluded1).toBe(true);
-
-        const source2 = { config: { executorId: 'default' } };
-        const isSourceIncluded2 = shouldIncludeSource(source2, IngestionSourceType.ALL);
-        expect(isSourceIncluded2).toBe(true);
-    });
-});
diff --git a/datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx b/datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx
index 87d632bb228b5..f533890e17fa5 100644
--- a/datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx
@@ -1,7 +1,7 @@
 import React from 'react';
 import { Alert } from 'antd';
 
-const CSV_FORMAT_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/csv';
+const CSV_FORMAT_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher';
 
 export const CSVInfo = () => {
     const link = (
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx
index 4199658568b9a..528c1fe4a6a9e 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx
@@ -118,7 +118,7 @@ function RecipeForm(props: Props) {
         },
     });
     const secrets =
-        data?.listSecrets?.secrets.sort((secretA, secretB) => secretA.name.localeCompare(secretB.name)) || [];
+        data?.listSecrets?.secrets?.sort((secretA, secretB) => secretA.name.localeCompare(secretB.name)) || [];
     const [form] = Form.useForm();
 
     function updateFormValues(changedValues: any, allValues: any) {
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx
index 73acfbcc3fc4e..7e49eb836d440 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx
@@ -102,7 +102,7 @@ function SecretField({ field, secrets, removeMargin, updateFormValue, refetchSec
         >
             <AutoComplete
                 placeholder={field.placeholder}
-                filterOption={(input, option) => !!option?.value.toLowerCase().includes(input.toLowerCase())}
+                filterOption={(input, option) => !!option?.value?.toLowerCase().includes(input.toLowerCase())}
                 notFoundContent={<>No secrets found</>}
                 options={options}
                 dropdownRender={(menu) => {
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionModal.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionModal.tsx
index 3a0bc47fa7d26..049e30ba5b141 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionModal.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionModal.tsx
@@ -158,9 +158,9 @@ function TestConnectionModal({
                     {testConnectionResult?.basic_connectivity && (
                         <ConnectionCapabilityView
                             capability="Basic Connectivity"
-                            displayMessage={testConnectionResult?.basic_connectivity.failure_reason}
-                            success={testConnectionResult?.basic_connectivity.capable}
-                            tooltipMessage={testConnectionResult?.basic_connectivity.mitigation_message}
+                            displayMessage={testConnectionResult?.basic_connectivity?.failure_reason}
+                            success={testConnectionResult?.basic_connectivity?.capable}
+                            tooltipMessage={testConnectionResult?.basic_connectivity?.mitigation_message}
                             number={1}
                         />
                     )}
diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts
index b67ca388c1054..58525b3e88f97 100644
--- a/datahub-web-react/src/app/ingest/source/builder/constants.ts
+++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts
@@ -19,6 +19,7 @@ import clickhouseLogo from '../../../../images/clickhouselogo.png';
 import cockroachdbLogo from '../../../../images/cockroachdblogo.png';
 import trinoLogo from '../../../../images/trinologo.png';
 import dbtLogo from '../../../../images/dbtlogo.png';
+import dremioLogo from '../../../../images/dremiologo.png';
 import druidLogo from '../../../../images/druidlogo.png';
 import elasticsearchLogo from '../../../../images/elasticsearchlogo.png';
 import feastLogo from '../../../../images/feastlogo.png';
@@ -35,6 +36,9 @@ import csvLogo from '../../../../images/csv-logo.png';
 import qlikLogo from '../../../../images/qliklogo.png';
 import sigmaLogo from '../../../../images/sigmalogo.png';
 import sacLogo from '../../../../images/saclogo.svg';
+import cassandraLogo from '../../../../images/cassandralogo.png';
+import datahubLogo from '../../../../images/datahublogo.png';
+import neo4j from '../../../../images/neo4j.png';
 
 export const ATHENA = 'athena';
 export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`;
@@ -51,6 +55,8 @@ export const COCKROACHDB = 'cockroachdb';
 export const COCKROACHDB_URN = `urn:li:dataPlatform:${COCKROACHDB}`;
 export const DBT = 'dbt';
 export const DBT_URN = `urn:li:dataPlatform:${DBT}`;
+export const DREMIO = 'dremio';
+export const DREMIO_URN = `urn:li:dataPlatform:${DREMIO}`;
 export const DRUID = 'druid';
 export const DRUID_URN = `urn:li:dataPlatform:${DRUID}`;
 export const DYNAMODB = 'dynamodb';
@@ -125,6 +131,15 @@ export const SIGMA = 'sigma';
 export const SIGMA_URN = `urn:li:dataPlatform:${SIGMA}`;
 export const SAC = 'sac';
 export const SAC_URN = `urn:li:dataPlatform:${SAC}`;
+export const CASSANDRA = 'cassandra';
+export const CASSANDRA_URN = `urn:li:dataPlatform:${CASSANDRA}`;
+export const DATAHUB = 'datahub';
+export const DATAHUB_GC = 'datahub-gc';
+export const DATAHUB_LINEAGE_FILE = 'datahub-lineage-file';
+export const DATAHUB_BUSINESS_GLOSSARY = 'datahub-business-glossary';
+export const DATAHUB_URN = `urn:li:dataPlatform:${DATAHUB}`;
+export const NEO4J = 'neo4j';
+export const NEO4J_URN = `urn:li:dataPlatform:${NEO4J}`;
 
 export const PLATFORM_URN_TO_LOGO = {
     [ATHENA_URN]: athenaLogo,
@@ -133,6 +148,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [CLICKHOUSE_URN]: clickhouseLogo,
     [COCKROACHDB_URN]: cockroachdbLogo,
     [DBT_URN]: dbtLogo,
+    [DREMIO_URN]: dremioLogo,
     [DRUID_URN]: druidLogo,
     [DYNAMODB_URN]: dynamodbLogo,
     [ELASTICSEARCH_URN]: elasticsearchLogo,
@@ -165,6 +181,9 @@ export const PLATFORM_URN_TO_LOGO = {
     [QLIK_SENSE_URN]: qlikLogo,
     [SIGMA_URN]: sigmaLogo,
     [SAC_URN]: sacLogo,
+    [CASSANDRA_URN]: cassandraLogo,
+    [DATAHUB_URN]: datahubLogo,
+    [NEO4J_URN]: neo4j,
 };
 
 export const SOURCE_TO_PLATFORM_URN = {
@@ -178,5 +197,7 @@ export const SOURCE_TO_PLATFORM_URN = {
     [SNOWFLAKE_USAGE]: SNOWFLAKE_URN,
     [STARBURST_TRINO_USAGE]: TRINO_URN,
     [DBT_CLOUD]: DBT_URN,
-    [VERTICA]: VERTICA_URN,
+    [DATAHUB_GC]: DATAHUB_URN,
+    [DATAHUB_LINEAGE_FILE]: DATAHUB_URN,
+    [DATAHUB_BUSINESS_GLOSSARY]: DATAHUB_URN,
 };
diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
index bb1c1a10ea6e5..102cce0f491e3 100644
--- a/datahub-web-react/src/app/ingest/source/builder/sources.json
+++ b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -181,7 +181,7 @@
         "displayName": "Athena",
         "description": "Import Schemas, Tables, Views, and lineage to S3 from Athena.",
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/athena/",
-        "recipe": "source:\n  type: athena\n  config:\n    # Coordinates\n    aws_region: my_aws_region\n    work_group: primary\n\n    # Options\n    s3_staging_dir: \"s3://my_staging_athena_results_bucket/results/\""
+        "recipe": "source:\n  type: athena\n  config:\n    # AWS Keys (Optional - Required only if local aws credentials are not set)\n    username: aws_access_key_id\n    password: aws_secret_access_key\n    # Coordinates\n    aws_region: my_aws_region\n    work_group: primary\n\n    # Options\n    s3_staging_dir: \"s3://my_staging_athena_results_bucket/results/\""
     },
     {
         "urn": "urn:li:dataPlatform:clickhouse",
@@ -284,7 +284,7 @@
         "name": "csv-enricher",
         "displayName": "CSV",
         "description": "Import metadata from a formatted CSV.",
-        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv'",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher",
         "recipe": "source: \n    type: csv-enricher \n    config: \n        # URL of your csv file to ingest \n        filename: \n        array_delimiter: '|' \n        delimiter: ',' \n        write_semantics: PATCH"
     },
     {
@@ -302,5 +302,36 @@
         "description": "Configure a custom recipe using YAML.",
         "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
         "recipe": "source:\n  type: <source-type>\n  config:\n    # Source-type specifics config\n    <source-configs>"
+    },
+    {
+        "urn": "urn:li:dataPlatform:dremio",
+        "name": "dremio",
+        "displayName": "Dremio",
+        "description": "Import Spaces, Sources, Tables and statistics from Dremio.",
+        "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
+        "recipe": "source:\n    type: dremio\n    config:\n        # Coordinates\n        hostname: null\n        port: null\n        #true if https, otherwise false\n        tls: true\n\n        #For cloud instance\n        #is_dremio_cloud: True\n        #dremio_cloud_project_id: <project_id>\n\n        #Credentials with personal access token\n        authentication_method: PAT\n        password: pass\n\n        #Or Credentials with basic auth\n        #authentication_method: password\n        #username: null\n        #password: null\n\n       ingest_owner: true\n\n        stateful_ingestion:\n            enabled: true"
+    },
+    {
+        "urn": "urn:li:dataPlatform:cassandra",
+        "name": "cassandra",
+        "displayName": "CassandraDB",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/cassandra",
+        "recipe": "source:\n  type: cassandra\n  config:\n    # Credentials for on prem cassandra\n    contact_point: localhost\n    port: 9042\n    username: admin\n    password: password\n\n    # Or\n    # Credentials Astra Cloud\n    #cloud_config:\n    #  secure_connect_bundle: Path to Secure Connect Bundle (.zip)\n    #  token: Application Token\n\n    # Optional Allow / Deny extraction of particular keyspaces.\n    keyspace_pattern:\n      allow: [.*]\n\n    # Optional Allow / Deny extraction of particular tables.\n    table_pattern:\n      allow: [.*]"
+    },
+    {
+        "urn": "urn:li:dataPlatform:iceberg",
+        "name": "iceberg",
+        "displayName": "Iceberg",
+        "description": "Ingest databases and tables from any Iceberg catalog implementation",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/iceberg",
+        "recipe": "source:\n type: \"iceberg\"\n config:\n   env: dev\n   # each thread will open internet connections to fetch manifest files independently, \n   # this value needs to be adjusted with ulimit\n   processing_threads: 1 \n   # a single catalog definition with a form of a dictionary\n   catalog: \n     demo: # name of the catalog\n       type: \"rest\" # other types are available\n       uri: \"uri\"\n       s3.access-key-id: \"access-key\"\n       s3.secret-access-key: \"secret-access-key\"\n       s3.region: \"aws-region\"\n   profiling:\n     enabled: false\n"
+    },
+    {
+        "urn": "urn:li:dataPlatform:neo4j",
+        "name": "neo4j",
+        "displayName": "Neo4j",
+        "description": "Import Nodes and Relationships from Neo4j.",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/neo4j/",
+        "recipe": "source:\n    type: 'neo4j'\n    config:\n        uri: 'neo4j+ssc://host:7687'\n        username: 'neo4j'\n        password: 'password'\n        env: 'PROD'\n\nsink:\n  type: \"datahub-rest\"\n  config:\n    server: 'http://localhost:8080'"
     }
 ]
diff --git a/datahub-web-react/src/app/ingest/source/conf/csv/csv.ts b/datahub-web-react/src/app/ingest/source/conf/csv/csv.ts
index e1dc22c086fb4..e4cdee717923c 100644
--- a/datahub-web-react/src/app/ingest/source/conf/csv/csv.ts
+++ b/datahub-web-react/src/app/ingest/source/conf/csv/csv.ts
@@ -15,7 +15,7 @@ const csvConfig: SourceConfig = {
     type: 'csv-enricher',
     placeholderRecipe,
     displayName: 'CSV',
-    docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/csv',
+    docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher',
     logoUrl: csvLogo,
 };
 
diff --git a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
index c8d4188886750..f56eb06b6af14 100644
--- a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
+++ b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
@@ -17,6 +17,7 @@ import {
     getStructuredReport,
     RUNNING,
     SUCCESS,
+    SUCCEEDED_WITH_WARNINGS,
 } from '../utils';
 import { ExecutionRequestResult } from '../../../../types.generated';
 import { StructuredReport } from './reporting/StructuredReport';
@@ -128,7 +129,7 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
         downloadFile(output, `exec-${urn}.log`);
     };
 
-    const logs = (showExpandedLogs && output) || output?.split('\n').slice(0, 5).join('\n');
+    const logs = (showExpandedLogs && output) || output?.split('\n')?.slice(0, 5)?.join('\n');
     const result = data?.executionRequest?.result as Partial<ExecutionRequestResult>;
     const status = getIngestionSourceStatus(result);
 
@@ -155,14 +156,14 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
         (status && <Typography.Text type="secondary">{getExecutionRequestSummaryText(status)}</Typography.Text>) ||
         undefined;
 
-    const recipeJson = data?.executionRequest?.input.arguments?.find((arg) => arg.key === 'recipe')?.value;
+    const recipeJson = data?.executionRequest?.input?.arguments?.find((arg) => arg.key === 'recipe')?.value;
     let recipeYaml: string;
     try {
         recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
     } catch (e) {
         recipeYaml = '';
     }
-    const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n').slice(0, 5).join('\n');
+    const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n')?.slice(0, 5)?.join('\n');
 
     const areLogsExpandable = output?.split(/\r\n|\r|\n/)?.length > 5;
     const isRecipeExpandable = recipeYaml?.split(/\r\n|\r|\n/)?.length > 5;
@@ -190,7 +191,7 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
                     <SubHeaderParagraph>{resultSummaryText}</SubHeaderParagraph>
                     {structuredReport ? <StructuredReport report={structuredReport} /> : null}
                 </StatusSection>
-                {status === SUCCESS && (
+                {(status === SUCCESS || status === SUCCEEDED_WITH_WARNINGS) && (
                     <IngestedAssetsSection>
                         {data?.executionRequest?.id && <IngestedAssets id={data?.executionRequest?.id} />}
                     </IngestedAssetsSection>
diff --git a/datahub-web-react/src/app/ingest/source/executions/IngestionSourceExecutionList.tsx b/datahub-web-react/src/app/ingest/source/executions/IngestionSourceExecutionList.tsx
index 9c40a3334df17..526f6f08334f5 100644
--- a/datahub-web-react/src/app/ingest/source/executions/IngestionSourceExecutionList.tsx
+++ b/datahub-web-react/src/app/ingest/source/executions/IngestionSourceExecutionList.tsx
@@ -49,7 +49,7 @@ export const IngestionSourceExecutionList = ({ urn, isExpanded, lastRefresh, onR
     };
 
     function hasActiveExecution() {
-        return !!data?.ingestionSource?.executions?.executionRequests.find((request) =>
+        return !!data?.ingestionSource?.executions?.executionRequests?.find((request) =>
             isExecutionRequestActive(request as ExecutionRequest),
         );
     }
diff --git a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx
index d15f30bc03211..1cd4349f37d94 100644
--- a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx
+++ b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx
@@ -16,6 +16,7 @@ const StyledCollapse = styled(Collapse)<{ color: string }>`
         .ant-collapse-header {
             display: flex;
             align-items: center;
+            overflow: auto;
         }
 
         .ant-collapse-item {
diff --git a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx
index abf6ce2c676bc..91b6282088f46 100644
--- a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx
+++ b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx
@@ -17,7 +17,7 @@ const Title = styled.div`
     font-weight: bold;
 `;
 
-const Item = styled.div`
+const Item = styled.pre`
     padding: 6px;
     font-size: 12px;
     border-radius: 2px;
diff --git a/datahub-web-react/src/app/lineage/LineageEntityColumns.tsx b/datahub-web-react/src/app/lineage/LineageEntityColumns.tsx
index 281d6381741f8..4777c6fef4fd4 100644
--- a/datahub-web-react/src/app/lineage/LineageEntityColumns.tsx
+++ b/datahub-web-react/src/app/lineage/LineageEntityColumns.tsx
@@ -67,7 +67,7 @@ export default function LineageEntityColumns({ node, onHover }: Props) {
     }, [displayedFields, node?.data?.urn, setVisibleColumnsByUrn, previousDisplayedFields]);
 
     const hasColumnPagination =
-        (node.data.schemaMetadata?.fields && node.data.schemaMetadata?.fields.length > NUM_COLUMNS_PER_PAGE) ||
+        (node.data.schemaMetadata?.fields && node.data.schemaMetadata?.fields?.length > NUM_COLUMNS_PER_PAGE) ||
         (node.data.inputFields?.fields && node.data.inputFields.fields.length > NUM_COLUMNS_PER_PAGE);
 
     return (
diff --git a/datahub-web-react/src/app/lineage/LineageEntityNode.tsx b/datahub-web-react/src/app/lineage/LineageEntityNode.tsx
index fcf2198e7e004..994090059754c 100644
--- a/datahub-web-react/src/app/lineage/LineageEntityNode.tsx
+++ b/datahub-web-react/src/app/lineage/LineageEntityNode.tsx
@@ -19,6 +19,10 @@ import ManageLineageMenu from './manage/ManageLineageMenu';
 import { useGetLineageTimeParams } from './utils/useGetLineageTimeParams';
 import { EntityHealth } from '../entity/shared/containers/profile/header/EntityHealth';
 import { EntityType } from '../../types.generated';
+import StructuredPropertyBadge, {
+    MAX_PROP_BADGE_WIDTH,
+} from '../entity/shared/containers/profile/header/StructuredPropertyBadge';
+import { filterForAssetBadge } from '../entity/shared/containers/profile/header/utils';
 
 const CLICK_DELAY_THRESHOLD = 1000;
 const DRAG_DISTANCE_THRESHOLD = 20;
@@ -38,6 +42,11 @@ const MultilineTitleText = styled.p`
     word-break: break-all;
 `;
 
+const PropertyBadgeWrapper = styled.div`
+    display: flex;
+    justify-content: flex-end;
+`;
+
 export default function LineageEntityNode({
     node,
     isSelected,
@@ -150,6 +159,11 @@ export default function LineageEntityNode({
     const baseUrl = node.data.type && node.data.urn && entityRegistry.getEntityUrl(node.data.type, node.data.urn);
     const hasHealth = (health && baseUrl) || false;
 
+    const entityStructuredProps = node.data.structuredProperties;
+    const hasAssetBadge = entityStructuredProps?.properties?.find(filterForAssetBadge);
+    const siblingStructuredProps = node.data.siblingStructuredProperties;
+    const siblingHasAssetBadge = siblingStructuredProps?.properties?.find(filterForAssetBadge);
+
     return (
         <PointerGroup data-testid={`node-${node.data.urn}-${direction}`} top={node.x} left={node.y}>
             {unexploredHiddenChildren && (isHovered || isSelected) ? (
@@ -340,6 +354,32 @@ export default function LineageEntityNode({
                         />
                     </foreignObject>
                 )}
+                {hasAssetBadge && (
+                    <foreignObject
+                        x={-centerX - MAX_PROP_BADGE_WIDTH - 8}
+                        y={centerY - 15}
+                        width={MAX_PROP_BADGE_WIDTH}
+                        height={30}
+                        onClick={(e) => e.stopPropagation()}
+                    >
+                        <PropertyBadgeWrapper>
+                            <StructuredPropertyBadge structuredProperties={entityStructuredProps ?? undefined} />
+                        </PropertyBadgeWrapper>
+                    </foreignObject>
+                )}
+                {!hasAssetBadge && siblingHasAssetBadge && (
+                    <foreignObject
+                        x={-centerX - MAX_PROP_BADGE_WIDTH - 8}
+                        y={centerY - 15}
+                        width={MAX_PROP_BADGE_WIDTH}
+                        height={30}
+                        onClick={(e) => e.stopPropagation()}
+                    >
+                        <PropertyBadgeWrapper>
+                            <StructuredPropertyBadge structuredProperties={siblingStructuredProps ?? undefined} />
+                        </PropertyBadgeWrapper>
+                    </foreignObject>
+                )}
                 <Group>
                     <UnselectableText
                         dy="-1em"
diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx
index 2d42d164f6626..fbf305a034d4d 100644
--- a/datahub-web-react/src/app/lineage/LineageExplorer.tsx
+++ b/datahub-web-react/src/app/lineage/LineageExplorer.tsx
@@ -101,7 +101,7 @@ export default function LineageExplorer({ urn, type }: Props) {
 
     const maybeAddAsyncLoadedEntity = useCallback(
         (entityAndType: EntityAndType) => {
-            if (entityAndType?.entity.urn && !asyncEntities.get(entityAndType?.entity.urn)?.fullyFetched) {
+            if (entityAndType?.entity?.urn && !asyncEntities.get(entityAndType?.entity?.urn)?.fullyFetched) {
                 // record that we have added this entity
                 let newAsyncEntities = extendAsyncEntities(
                     fineGrainedMap,
diff --git a/datahub-web-react/src/app/lineage/manage/AddEntityEdge.tsx b/datahub-web-react/src/app/lineage/manage/AddEntityEdge.tsx
index d5041c63da1d3..a4057f97fe37a 100644
--- a/datahub-web-react/src/app/lineage/manage/AddEntityEdge.tsx
+++ b/datahub-web-react/src/app/lineage/manage/AddEntityEdge.tsx
@@ -104,7 +104,7 @@ export default function AddEntityEdge({
     }
 
     function selectEntity(urn: string) {
-        const resultEntities = autoCompleteResults?.autoCompleteForMultiple?.suggestions.flatMap(
+        const resultEntities = autoCompleteResults?.autoCompleteForMultiple?.suggestions?.flatMap(
             (suggestion) => suggestion.entities || [],
         );
         const selectedEntity = resultEntities?.find((entity) => entity.urn === urn);
diff --git a/datahub-web-react/src/app/lineage/types.ts b/datahub-web-react/src/app/lineage/types.ts
index 0edc3a3a7c039..8a25ba202a085 100644
--- a/datahub-web-react/src/app/lineage/types.ts
+++ b/datahub-web-react/src/app/lineage/types.ts
@@ -20,6 +20,7 @@ import {
     SchemaMetadata,
     SiblingProperties,
     Status,
+    StructuredProperties,
 } from '../../types.generated';
 
 export type EntitySelectParams = {
@@ -58,6 +59,7 @@ export type FetchedEntity = {
     inputFields?: InputFields;
     canEditLineage?: boolean;
     health?: Maybe<Health[]>;
+    structuredProperties?: Maybe<StructuredProperties>;
 };
 
 export type NodeData = {
@@ -82,6 +84,8 @@ export type NodeData = {
     upstreamRelationships?: Array<LineageRelationship>;
     downstreamRelationships?: Array<LineageRelationship>;
     health?: Maybe<Health[]>;
+    structuredProperties?: Maybe<StructuredProperties>;
+    siblingStructuredProperties?: Maybe<StructuredProperties>;
 };
 
 export type VizNode = {
diff --git a/datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts b/datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts
index c3483f229f7a0..bd4f07cae1117 100644
--- a/datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts
+++ b/datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts
@@ -149,7 +149,7 @@ export function filterColumns(
     setColumnsByUrn: (value: React.SetStateAction<Record<string, SchemaField[]>>) => void,
 ) {
     const formattedFilterText = filterText.toLocaleLowerCase();
-    const filteredFields = node.data.schemaMetadata?.fields.filter((field) =>
+    const filteredFields = node.data.schemaMetadata?.fields?.filter((field) =>
         field.fieldPath.toLocaleLowerCase().includes(formattedFilterText),
     );
     if (filteredFields) {
diff --git a/datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts b/datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts
index 12d4cca352bb3..bb9f29522d0cb 100644
--- a/datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts
+++ b/datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts
@@ -68,6 +68,7 @@ export default function constructFetchedNode(
             upstreamRelationships: fetchedNode?.upstreamRelationships || [],
             downstreamRelationships: fetchedNode?.downstreamRelationships || [],
             health: fetchedNode?.health,
+            structuredProperties: fetchedNode?.structuredProperties,
         };
 
         // eslint-disable-next-line no-param-reassign
diff --git a/datahub-web-react/src/app/lineage/utils/constructTree.ts b/datahub-web-react/src/app/lineage/utils/constructTree.ts
index 38a865ea9e093..4e94ad2813674 100644
--- a/datahub-web-react/src/app/lineage/utils/constructTree.ts
+++ b/datahub-web-react/src/app/lineage/utils/constructTree.ts
@@ -83,6 +83,8 @@ export default function constructTree(
     });
 
     const fetchedEntity = entityRegistry.getLineageVizConfig(entityAndType.type, entityAndType.entity);
+    const sibling = fetchedEntity?.siblings?.siblings?.[0];
+    const fetchedSiblingEntity = sibling ? entityRegistry.getLineageVizConfig(sibling.type, sibling) : null;
 
     const root: NodeData = {
         name: fetchedEntity?.name || '',
@@ -100,6 +102,8 @@ export default function constructTree(
         upstreamRelationships: fetchedEntity?.upstreamRelationships || [],
         downstreamRelationships: fetchedEntity?.downstreamRelationships || [],
         health: fetchedEntity?.health,
+        structuredProperties: fetchedEntity?.structuredProperties,
+        siblingStructuredProperties: fetchedSiblingEntity?.structuredProperties,
     };
     const lineageConfig = entityRegistry.getLineageVizConfig(entityAndType.type, entityAndType.entity);
     let updatedLineageConfig = { ...lineageConfig };
diff --git a/datahub-web-react/src/app/lineage/utils/layoutTree.ts b/datahub-web-react/src/app/lineage/utils/layoutTree.ts
index a972a62308f07..bc88337ccebdb 100644
--- a/datahub-web-react/src/app/lineage/utils/layoutTree.ts
+++ b/datahub-web-react/src/app/lineage/utils/layoutTree.ts
@@ -219,9 +219,9 @@ function drawColumnEdge({
     const targetFieldY = targetNode?.y || 0 + 3;
     let targetFieldX = (targetNode?.x || 0) + 35 + targetTitleHeight;
     // if currentNode is a dataJob, draw line to center of data job
-    if (targetNode?.data.type === EntityType.DataJob) {
+    if (targetNode?.data?.type === EntityType.DataJob) {
         targetFieldX = targetNode?.x || 0;
-    } else if (!collapsedColumnsNodes[targetNode?.data.urn || 'no-op']) {
+    } else if (!collapsedColumnsNodes[targetNode?.data?.urn || 'no-op']) {
         if (!visibleColumnsByUrn[targetUrn]?.has(targetField)) {
             targetFieldX =
                 (targetNode?.x || 0) +
@@ -299,19 +299,19 @@ function layoutColumnTree(
                 const fieldForwardEdges = fieldPathToEdges[sourceField];
 
                 const currentNode = nodesToRender.find((node) => node.data.urn === entityUrn);
-                const fields = columnsByUrn[currentNode?.data.urn || ''] || [];
+                const fields = columnsByUrn[currentNode?.data?.urn || ''] || [];
                 const fieldIndex = fields.findIndex((candidate) => candidate.fieldPath === sourceField) || 0;
 
                 const sourceTitleHeight = getTitleHeight(
-                    expandTitles ? currentNode?.data.expandedName || currentNode?.data.name : undefined,
+                    expandTitles ? currentNode?.data?.expandedName || currentNode?.data?.name : undefined,
                 );
 
                 const sourceFieldY = currentNode?.y || 0 + 1;
                 let sourceFieldX = (currentNode?.x || 0) + 30 + sourceTitleHeight;
                 // if currentNode is a dataJob, draw line from center of data job
-                if (currentNode?.data.type === EntityType.DataJob) {
+                if (currentNode?.data?.type === EntityType.DataJob) {
                     sourceFieldX = currentNode?.x || 0;
-                } else if (!collapsedColumnsNodes[currentNode?.data.urn || 'no-op']) {
+                } else if (!collapsedColumnsNodes[currentNode?.data?.urn || 'no-op']) {
                     if (!visibleColumnsByUrn[entityUrn]?.has(sourceField)) {
                         sourceFieldX =
                             (currentNode?.x || 0) +
@@ -331,9 +331,9 @@ function layoutColumnTree(
 
                 Object.keys(fieldForwardEdges || {}).forEach((targetUrn) => {
                     const targetNode = nodesToRender.find((node) => node.data.urn === targetUrn);
-                    const targetFields = columnsByUrn[targetNode?.data.urn || ''] || [];
+                    const targetFields = columnsByUrn[targetNode?.data?.urn || ''] || [];
                     const targetTitleHeight = getTitleHeight(
-                        expandTitles ? targetNode?.data.expandedName || targetNode?.data.name : undefined,
+                        expandTitles ? targetNode?.data?.expandedName || targetNode?.data?.name : undefined,
                     );
 
                     (fieldForwardEdges[targetUrn] || []).forEach((targetField) => {
diff --git a/datahub-web-react/src/app/lineage/utils/titleUtils.ts b/datahub-web-react/src/app/lineage/utils/titleUtils.ts
index 6bd4cfea0f09a..8bd0cbda55b33 100644
--- a/datahub-web-react/src/app/lineage/utils/titleUtils.ts
+++ b/datahub-web-react/src/app/lineage/utils/titleUtils.ts
@@ -124,10 +124,10 @@ function truncate(input, length) {
 function getLastTokenOfTitle(title?: string): string {
     if (!title) return '';
 
-    const lastToken = title?.split('.').slice(-1)[0];
+    const lastToken = title?.split('.')?.slice(-1)?.[0];
 
     // if the last token does not contain any content, the string should not be tokenized on `.`
-    if (lastToken.replace(/\s/g, '').length === 0) {
+    if (lastToken?.replace(/\s/g, '')?.length === 0) {
         return title;
     }
 
diff --git a/datahub-web-react/src/app/permissions/policy/PolicyActorForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyActorForm.tsx
index adb660d43f471..f351c6db3966f 100644
--- a/datahub-web-react/src/app/permissions/policy/PolicyActorForm.tsx
+++ b/datahub-web-react/src/app/permissions/policy/PolicyActorForm.tsx
@@ -58,7 +58,7 @@ export default function PolicyActorForm({ policyType, actors, setActors }: Props
         },
     });
     const ownershipTypes =
-        ownershipData?.listOwnershipTypes?.ownershipTypes.filter((type) => type.urn !== 'urn:li:ownershipType:none') ||
+        ownershipData?.listOwnershipTypes?.ownershipTypes?.filter((type) => type.urn !== 'urn:li:ownershipType:none') ||
         [];
     const ownershipTypesMap = Object.fromEntries(ownershipTypes.map((type) => [type.urn, type.info?.name]));
     // Toggle the "Owners" switch
diff --git a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
index 37349585fa4c9..1988fea349699 100644
--- a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
+++ b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
@@ -12,6 +12,7 @@ import {
     mapResourceTypeToDisplayName,
 } from './policyUtils';
 import AvatarsGroup from '../AvatarsGroup';
+import { RESOURCE_TYPE, RESOURCE_URN, TYPE, URN } from './constants';
 
 type PrivilegeOptionType = {
     type?: string;
@@ -72,10 +73,11 @@ export default function PolicyDetailsModal({ policy, open, onClose, privileges }
     const isMetadataPolicy = policy?.type === PolicyType.Metadata;
 
     const resources = convertLegacyResourceFilter(policy?.resources);
-    const resourceTypes = getFieldValues(resources?.filter, 'TYPE') || [];
+    const resourceTypes = getFieldValues(resources?.filter, TYPE, RESOURCE_TYPE) || [];
     const dataPlatformInstances = getFieldValues(resources?.filter, 'DATA_PLATFORM_INSTANCE') || [];
-    const resourceEntities = getFieldValues(resources?.filter, 'URN') || [];
-    const resourceFilterCondition = getFieldCondition(resources?.filter, 'URN') || PolicyMatchCondition.Equals;
+    const resourceEntities = getFieldValues(resources?.filter, URN, RESOURCE_URN) || [];
+    const resourceFilterCondition =
+        getFieldCondition(resources?.filter, URN, RESOURCE_URN) || PolicyMatchCondition.Equals;
     const domains = getFieldValues(resources?.filter, 'DOMAIN') || [];
 
     const {
diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
index 7a0de67f41419..414346c2776db 100644
--- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
+++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
@@ -28,6 +28,7 @@ import ClickOutside from '../../shared/ClickOutside';
 import { TagTermLabel } from '../../shared/tags/TagTermLabel';
 import { ENTER_KEY_CODE } from '../../shared/constants';
 import { useGetRecommendations } from '../../shared/recommendation';
+import { RESOURCE_TYPE, RESOURCE_URN, TYPE, URN } from './constants';
 
 type Props = {
     policyType: PolicyType;
@@ -102,8 +103,9 @@ export default function PolicyPrivilegeForm({
     } = useAppConfig();
 
     const resources: ResourceFilter = convertLegacyResourceFilter(maybeResources) || EMPTY_POLICY.resources;
-    const resourceTypes = getFieldValues(resources.filter, 'TYPE') || [];
-    const resourceEntities = getFieldValues(resources.filter, 'URN') || [];
+    // RESOURCE_TYPE and RESOURCE_URN are deprecated, but need to get them for backwards compatibility
+    const resourceTypes = getFieldValues(resources.filter, TYPE, RESOURCE_TYPE) || [];
+    const resourceEntities = getFieldValues(resources.filter, URN, RESOURCE_URN) || [];
 
     const getDisplayName = (entity) => {
         if (!entity) {
@@ -178,9 +180,14 @@ export default function PolicyPrivilegeForm({
         const filter = resources.filter || {
             criteria: [],
         };
+        // remove the deprecated RESOURCE_TYPE field and replace with TYPE field
+        const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_TYPE, []);
         setResources({
             ...resources,
-            filter: setFieldValues(filter, 'TYPE', [...resourceTypes, createCriterionValue(selectedResourceType)]),
+            filter: setFieldValues(filterWithoutDeprecatedField, TYPE, [
+                ...resourceTypes,
+                createCriterionValue(selectedResourceType),
+            ]),
         });
     };
 
@@ -188,11 +195,13 @@ export default function PolicyPrivilegeForm({
         const filter = resources.filter || {
             criteria: [],
         };
+        // remove the deprecated RESOURCE_TYPE field and replace with TYPE field
+        const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_TYPE, []);
         setResources({
             ...resources,
             filter: setFieldValues(
-                filter,
-                'TYPE',
+                filterWithoutDeprecatedField,
+                TYPE,
                 resourceTypes?.filter((criterionValue) => criterionValue.value !== deselectedResourceType),
             ),
         });
@@ -203,9 +212,11 @@ export default function PolicyPrivilegeForm({
         const filter = resources.filter || {
             criteria: [],
         };
+        // remove the deprecated RESOURCE_URN field and replace with URN field
+        const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_URN, []);
         setResources({
             ...resources,
-            filter: setFieldValues(filter, 'URN', [
+            filter: setFieldValues(filterWithoutDeprecatedField, URN, [
                 ...resourceEntities,
                 createCriterionValueWithEntity(
                     resource,
@@ -220,11 +231,13 @@ export default function PolicyPrivilegeForm({
         const filter = resources.filter || {
             criteria: [],
         };
+        // remove the deprecated RESOURCE_URN field and replace with URN field
+        const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_URN, []);
         setResources({
             ...resources,
             filter: setFieldValues(
-                filter,
-                'URN',
+                filterWithoutDeprecatedField,
+                URN,
                 resourceEntities?.filter((criterionValue) => criterionValue.value !== resource),
             ),
         });
diff --git a/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx b/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx
index 1c9884e5fcf09..eae735b3477f0 100644
--- a/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx
+++ b/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx
@@ -1,4 +1,12 @@
-import { addOrUpdatePoliciesInList, updateListPoliciesCache, removeFromListPoliciesCache } from '../policyUtils';
+import { PolicyMatchCondition } from '../../../../types.generated';
+import {
+    addOrUpdatePoliciesInList,
+    updateListPoliciesCache,
+    removeFromListPoliciesCache,
+    getFieldValues,
+    getFieldCondition,
+    setFieldValues,
+} from '../policyUtils';
 
 // Mock the Apollo Client readQuery and writeQuery methods
 const mockReadQuery = vi.fn();
@@ -103,3 +111,153 @@ describe('removeFromListPoliciesCache', () => {
         });
     });
 });
+
+describe('getFieldValues', () => {
+    it('should get field values for a given field', () => {
+        const filter = {
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'TYPE',
+                    values: [{ value: 'dataset' }, { value: 'dataJob' }],
+                },
+            ],
+        };
+
+        expect(getFieldValues(filter, 'TYPE')).toMatchObject([{ value: 'dataset' }, { value: 'dataJob' }]);
+    });
+
+    it('should get field values for a alternate field (for deprecated fields)', () => {
+        const filter = {
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'RESOURCE_TYPE',
+                    values: [{ value: 'dataset' }, { value: 'dataJob' }],
+                },
+            ],
+        };
+
+        expect(getFieldValues(filter, 'TYPE', 'RESOURCE_TYPE')).toMatchObject([
+            { value: 'dataset' },
+            { value: 'dataJob' },
+        ]);
+    });
+
+    it('should get field values for main field with alternative field given and has values', () => {
+        const filter = {
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'RESOURCE_TYPE',
+                    values: [{ value: 'container' }, { value: 'dataFlow' }],
+                },
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'TYPE',
+                    values: [{ value: 'dataset' }, { value: 'dataJob' }],
+                },
+            ],
+        };
+
+        // should only return values from main field
+        expect(getFieldValues(filter, 'TYPE', 'RESOURCE_TYPE')).toMatchObject([
+            { value: 'dataset' },
+            { value: 'dataJob' },
+        ]);
+    });
+});
+
+describe('getFieldCondition', () => {
+    it('should get field values for a given field', () => {
+        const filter = {
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'TYPE',
+                    values: [{ value: 'dataset' }],
+                },
+            ],
+        };
+
+        expect(getFieldCondition(filter, 'TYPE')).toBe(PolicyMatchCondition.Equals);
+    });
+
+    it('should get field values for a alternate field (for deprecated fields)', () => {
+        const filter = {
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'RESOURCE_TYPE',
+                    values: [{ value: 'dataset' }],
+                },
+            ],
+        };
+
+        expect(getFieldCondition(filter, 'TYPE', 'RESOURCE_TYPE')).toBe(PolicyMatchCondition.Equals);
+    });
+
+    it('should get field values for main field with alternative field given and has values', () => {
+        const filter = {
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.StartsWith,
+                    field: 'RESOURCE_TYPE',
+                    values: [{ value: 'container' }, { value: 'dataFlow' }],
+                },
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'TYPE',
+                    values: [{ value: 'dataset' }],
+                },
+            ],
+        };
+
+        // should only return values from main field
+        expect(getFieldCondition(filter, 'TYPE', 'RESOURCE_TYPE')).toBe(PolicyMatchCondition.Equals);
+    });
+});
+describe('setFieldValues', () => {
+    it('should remove a field if you pass in an empty array', () => {
+        const filter = {
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'RESOURCE_TYPE',
+                    values: [{ value: 'dataset' }],
+                },
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'TYPE',
+                    values: [{ value: 'dataJob' }],
+                },
+            ],
+        };
+
+        expect(setFieldValues(filter, 'RESOURCE_TYPE', [])).toMatchObject({
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'TYPE',
+                    values: [{ value: 'dataJob' }],
+                },
+            ],
+        });
+    });
+
+    it('should set values for a field properly', () => {
+        const filter = {
+            criteria: [],
+        };
+
+        expect(setFieldValues(filter, 'TYPE', [{ value: 'dataFlow' }])).toMatchObject({
+            criteria: [
+                {
+                    condition: PolicyMatchCondition.Equals,
+                    field: 'TYPE',
+                    values: [{ value: 'dataFlow' }],
+                },
+            ],
+        });
+    });
+});
diff --git a/datahub-web-react/src/app/permissions/policy/constants.ts b/datahub-web-react/src/app/permissions/policy/constants.ts
new file mode 100644
index 0000000000000..cdd20bf9b50d6
--- /dev/null
+++ b/datahub-web-react/src/app/permissions/policy/constants.ts
@@ -0,0 +1,4 @@
+export const TYPE = 'TYPE';
+export const RESOURCE_TYPE = 'RESOURCE_TYPE';
+export const URN = 'URN';
+export const RESOURCE_URN = 'RESOURCE_URN';
diff --git a/datahub-web-react/src/app/permissions/policy/policyUtils.ts b/datahub-web-react/src/app/permissions/policy/policyUtils.ts
index b71a38f80fc25..d6221a0a9293a 100644
--- a/datahub-web-react/src/app/permissions/policy/policyUtils.ts
+++ b/datahub-web-react/src/app/permissions/policy/policyUtils.ts
@@ -114,12 +114,28 @@ export const convertLegacyResourceFilter = (resourceFilter: Maybe<ResourceFilter
     };
 };
 
-export const getFieldValues = (filter: Maybe<PolicyMatchFilter> | undefined, resourceFieldType: string) => {
-    return filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.values || [];
+export const getFieldValues = (
+    filter: Maybe<PolicyMatchFilter> | undefined,
+    resourceFieldType: string,
+    alternateResourceFieldType?: string,
+) => {
+    return (
+        filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.values ||
+        filter?.criteria?.find((criterion) => criterion.field === alternateResourceFieldType)?.values ||
+        []
+    );
 };
 
-export const getFieldCondition = (filter: Maybe<PolicyMatchFilter> | undefined, resourceFieldType: string) => {
-    return filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.condition || null;
+export const getFieldCondition = (
+    filter: Maybe<PolicyMatchFilter> | undefined,
+    resourceFieldType: string,
+    alternateResourceFieldType?: string,
+) => {
+    return (
+        filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.condition ||
+        filter?.criteria?.find((criterion) => criterion.field === alternateResourceFieldType)?.condition ||
+        null
+    );
 };
 
 export const getFieldValuesOfTags = (filter: Maybe<PolicyMatchFilter> | undefined, resourceFieldType: string) => {
diff --git a/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx b/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
index 7c2cec63071ad..151d23300c7ba 100644
--- a/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
+++ b/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
@@ -176,7 +176,7 @@ export const ManageRoles = () => {
             render: (_: any, record: any) => {
                 return (
                     <>
-                        {(record?.users.length && (
+                        {(record?.users?.length && (
                             <AvatarsGroup
                                 users={record?.users}
                                 groups={record?.resolvedGroups}
@@ -217,8 +217,8 @@ export const ManageRoles = () => {
         type: role?.type,
         description: role?.description,
         name: role?.name,
-        users: role?.users?.relationships.map((relationship) => relationship.entity as CorpUser),
-        policies: role?.policies?.relationships.map((relationship) => relationship.entity as DataHubPolicy),
+        users: role?.users?.relationships?.map((relationship) => relationship.entity as CorpUser),
+        policies: role?.policies?.relationships?.map((relationship) => relationship.entity as DataHubPolicy),
     }));
 
     return (
diff --git a/datahub-web-react/src/app/permissions/roles/RoleDetailsModal.tsx b/datahub-web-react/src/app/permissions/roles/RoleDetailsModal.tsx
index 7bca62c040c2d..6dc7ebbe1407c 100644
--- a/datahub-web-react/src/app/permissions/roles/RoleDetailsModal.tsx
+++ b/datahub-web-react/src/app/permissions/roles/RoleDetailsModal.tsx
@@ -45,8 +45,8 @@ export default function RoleDetailsModal({ role, open, onClose }: Props) {
 
     const castedRole = role as any;
 
-    const users = castedRole?.users?.relationships.map((relationship) => relationship.entity as CorpUser);
-    const policies = castedRole?.policies?.relationships.map((relationship) => relationship.entity as DataHubPolicy);
+    const users = castedRole?.users?.relationships?.map((relationship) => relationship.entity as CorpUser);
+    const policies = castedRole?.policies?.relationships?.map((relationship) => relationship.entity as DataHubPolicy);
 
     return (
         <Modal title={role?.name} open={open} onCancel={onClose} closable width={800} footer={actionButtons}>
diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx
index ee29e92e06a22..a19862e83ae51 100644
--- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx
+++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx
@@ -9,6 +9,7 @@ import {
     GlossaryTerms,
     SearchInsight,
     Container,
+    Dataset,
     ParentContainersResult,
     Maybe,
     CorpUser,
@@ -36,6 +37,8 @@ import { DataProductLink } from '../shared/tags/DataProductLink';
 import { EntityHealth } from '../entity/shared/containers/profile/header/EntityHealth';
 import SearchTextHighlighter from '../search/matches/SearchTextHighlighter';
 import { getUniqueOwners } from './utils';
+import StructuredPropertyBadge from '../entity/shared/containers/profile/header/StructuredPropertyBadge';
+import { usePreviewData } from '../entity/shared/PreviewContext';
 
 const PreviewContainer = styled.div`
     display: flex;
@@ -65,6 +68,7 @@ const TitleContainer = styled.div`
 const EntityTitleContainer = styled.div`
     display: flex;
     align-items: center;
+    gap: 8px;
 `;
 
 const EntityTitle = styled(Typography.Text)<{ $titleSizePx?: number }>`
@@ -74,7 +78,6 @@ const EntityTitle = styled(Typography.Text)<{ $titleSizePx?: number }>`
     }
 
     &&& {
-        margin-right 8px;
         font-size: ${(props) => props.$titleSizePx || 16}px;
         font-weight: 600;
         vertical-align: middle;
@@ -196,6 +199,7 @@ interface Props {
     previewType?: Maybe<PreviewType>;
     paths?: EntityPath[];
     health?: Health[];
+    parentDataset?: Dataset;
 }
 
 export default function DefaultPreviewCard({
@@ -238,10 +242,12 @@ export default function DefaultPreviewCard({
     previewType,
     paths,
     health,
+    parentDataset,
 }: Props) {
     // sometimes these lists will be rendered inside an entity container (for example, in the case of impact analysis)
     // in those cases, we may want to enrich the preview w/ context about the container entity
     const { entityData } = useEntityData();
+    const previewData = usePreviewData();
     const insightViews: Array<ReactNode> = [
         ...(insights?.map((insight) => (
             <>
@@ -284,6 +290,7 @@ export default function DefaultPreviewCard({
                         parentEntities={parentEntities}
                         parentContainersRef={contentRef}
                         areContainersTruncated={isContentTruncated}
+                        parentDataset={parentDataset}
                     />
                     <EntityTitleContainer>
                         <Link to={url}>
@@ -301,6 +308,7 @@ export default function DefaultPreviewCard({
                             <DeprecationPill deprecation={deprecation} urn="" showUndeprecate={false} />
                         )}
                         {health && health.length > 0 ? <EntityHealth baseUrl={url} health={health} /> : null}
+                        <StructuredPropertyBadge structuredProperties={previewData?.structuredProperties} />
                         {externalUrl && (
                             <ExternalUrlButton
                                 externalUrl={externalUrl}
diff --git a/datahub-web-react/src/app/search/AdvancedFilterSelectValueModal.tsx b/datahub-web-react/src/app/search/AdvancedFilterSelectValueModal.tsx
index 056aca56c1313..43b4388e92ba1 100644
--- a/datahub-web-react/src/app/search/AdvancedFilterSelectValueModal.tsx
+++ b/datahub-web-react/src/app/search/AdvancedFilterSelectValueModal.tsx
@@ -47,7 +47,7 @@ export const AdvancedFilterSelectValueModal = ({
                 urns={[]}
                 defaultValues={initialValues?.map((urn) => ({
                     urn,
-                    entity: facet?.aggregations.find((aggregation) => aggregation.value === urn)?.entity,
+                    entity: facet?.aggregations?.find((aggregation) => aggregation.value === urn)?.entity,
                 }))}
                 onCloseModal={onCloseModal}
                 hideOwnerType
@@ -66,7 +66,7 @@ export const AdvancedFilterSelectValueModal = ({
                 defaultValue={
                     initialValues?.map((urn) => ({
                         urn,
-                        entity: facet?.aggregations.find((aggregation) => aggregation.value === urn)?.entity,
+                        entity: facet?.aggregations?.find((aggregation) => aggregation.value === urn)?.entity,
                     }))?.[0]
                 }
                 onCloseModal={onCloseModal}
@@ -84,7 +84,7 @@ export const AdvancedFilterSelectValueModal = ({
                 titleOverride="Select Container"
                 defaultValues={initialValues?.map((urn) => ({
                     urn,
-                    entity: facet?.aggregations.find((aggregation) => aggregation.value === urn)?.entity,
+                    entity: facet?.aggregations?.find((aggregation) => aggregation.value === urn)?.entity,
                 }))}
                 onCloseModal={onCloseModal}
                 onOkOverride={(containerUrns) => {
@@ -100,7 +100,7 @@ export const AdvancedFilterSelectValueModal = ({
             <SelectPlatformModal
                 defaultValues={initialValues?.map((urn) => ({
                     urn,
-                    entity: facet?.aggregations.find((aggregation) => aggregation.value === urn)?.entity,
+                    entity: facet?.aggregations?.find((aggregation) => aggregation.value === urn)?.entity,
                 }))}
                 titleOverride="Select Platform"
                 onCloseModal={onCloseModal}
@@ -195,7 +195,7 @@ export const AdvancedFilterSelectValueModal = ({
                 }}
                 defaultValues={initialValues?.map((urn) => ({
                     urn,
-                    entity: facet?.aggregations.find((aggregation) => aggregation.value === urn)?.entity,
+                    entity: facet?.aggregations?.find((aggregation) => aggregation.value === urn)?.entity,
                 }))}
             />
         );
@@ -219,7 +219,7 @@ export const AdvancedFilterSelectValueModal = ({
                 }}
                 defaultValues={initialValues?.map((urn) => ({
                     urn,
-                    entity: facet?.aggregations.find((aggregation) => aggregation.value === urn)?.entity,
+                    entity: facet?.aggregations?.find((aggregation) => aggregation.value === urn)?.entity,
                 }))}
             />
         );
diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx
index 7568e4ece8403..15bdf62eadf79 100644
--- a/datahub-web-react/src/app/search/SearchBar.tsx
+++ b/datahub-web-react/src/app/search/SearchBar.tsx
@@ -188,7 +188,7 @@ export const SearchBar = ({
 
     const emptyQueryOptions = useMemo(() => {
         const moduleOptions =
-            data?.listRecommendations?.modules.map((module) => ({
+            data?.listRecommendations?.modules?.map((module) => ({
                 label: <EntityTypeLabel>{module.title}</EntityTypeLabel>,
                 options: [...module.content.map((content) => renderRecommendedQuery(content.value))],
             })) || [];
diff --git a/datahub-web-react/src/app/search/SearchFilterLabel.tsx b/datahub-web-react/src/app/search/SearchFilterLabel.tsx
index 5a0e75cc2ae1c..2ed2d4608de0d 100644
--- a/datahub-web-react/src/app/search/SearchFilterLabel.tsx
+++ b/datahub-web-react/src/app/search/SearchFilterLabel.tsx
@@ -23,6 +23,9 @@ import CustomAvatar from '../shared/avatar/CustomAvatar';
 import { IconStyleType } from '../entity/Entity';
 import { formatNumber } from '../shared/formatNumber';
 import useGetBrowseV2LabelOverride from './filters/useGetBrowseV2LabelOverride';
+import { getParentEntities } from './filters/utils';
+import { ParentWrapper } from '../entity/shared/containers/profile/sidebar/Container/ContainerSelectModal';
+import ParentEntities from './filters/ParentEntities';
 
 type Props = {
     field: string;
@@ -157,11 +160,17 @@ export const SearchFilterLabel = ({ field, value, entity, count, hideCount }: Pr
     if (entity?.type === EntityType.Container) {
         const container = entity as Container;
         const displayName = entityRegistry.getDisplayName(EntityType.Container, container);
+        const parentEntities: Entity[] = getParentEntities(container as Entity) || [];
         const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName;
         return (
             <Tooltip title={displayName}>
                 {!!container.platform?.properties?.logoUrl && (
-                    <PreviewImage src={container.platform?.properties?.logoUrl} alt={container.properties?.name} />
+                    <>
+                        <ParentWrapper style={{ width: '200px' }}>
+                            <ParentEntities parentEntities={parentEntities} />
+                        </ParentWrapper>
+                        <PreviewImage src={container.platform?.properties?.logoUrl} alt={container.properties?.name} />
+                    </>
                 )}
                 <span>
                     {truncatedDisplayName}
diff --git a/datahub-web-react/src/app/search/SearchPage.tsx b/datahub-web-react/src/app/search/SearchPage.tsx
index fa0643742e376..aa83e739f2d77 100644
--- a/datahub-web-react/src/app/search/SearchPage.tsx
+++ b/datahub-web-react/src/app/search/SearchPage.tsx
@@ -59,7 +59,7 @@ export const SearchPage = () => {
                 orFilters,
                 viewUrn,
                 sortInput,
-                searchFlags: { getSuggestions: true },
+                searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true },
             },
         },
         fetchPolicy: 'cache-and-network',
diff --git a/datahub-web-react/src/app/search/SearchablePage.tsx b/datahub-web-react/src/app/search/SearchablePage.tsx
index 8a2b0b35ec31f..8e347671079b9 100644
--- a/datahub-web-react/src/app/search/SearchablePage.tsx
+++ b/datahub-web-react/src/app/search/SearchablePage.tsx
@@ -78,7 +78,11 @@ export const SearchablePage = ({ onSearch, onAutoComplete, children }: Props) =>
             const formattedPath = location.pathname
                 .split('/')
                 .filter((word) => word !== '')
-                .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+                .map((rawWord) => {
+                    // ie. personal-notifications -> Personal Notifications
+                    const words = rawWord.split('-');
+                    return words.map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(' ');
+                })
                 .join(' | ');
 
             if (formattedPath) {
diff --git a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx
index 416b04403723f..877efb55fcf82 100644
--- a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx
+++ b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx
@@ -4,7 +4,12 @@ import { FacetFilterInput, FacetMetadata } from '../../types.generated';
 import { FilterScenarioType } from './filters/render/types';
 import { useFilterRendererRegistry } from './filters/render/useFilterRenderer';
 import { SimpleSearchFilter } from './SimpleSearchFilter';
-import { ENTITY_FILTER_NAME, ENTITY_INDEX_FILTER_NAME, LEGACY_ENTITY_FILTER_NAME } from './utils/constants';
+import {
+    DEGREE_FILTER_NAME,
+    ENTITY_FILTER_NAME,
+    ENTITY_INDEX_FILTER_NAME,
+    LEGACY_ENTITY_FILTER_NAME,
+} from './utils/constants';
 
 const TOP_FILTERS = ['degree', ENTITY_FILTER_NAME, 'platform', 'tags', 'glossaryTerms', 'domains', 'owners'];
 
@@ -43,6 +48,15 @@ export const SimpleSearchFilters = ({ facets, selectedFilters, onFilterSelect, l
                           : filter,
                   )
                   .filter((filter) => filter.field !== field || !(filter.values?.length === 0));
+
+        // Do not let user unselect all degree filters
+        if (field === DEGREE_FILTER_NAME && !selected) {
+            const hasDegreeFilter = newFilters.find((filter) => filter.field === DEGREE_FILTER_NAME);
+            if (!hasDegreeFilter) {
+                return;
+            }
+        }
+
         setCachedProps({ ...cachedProps, selectedFilters: newFilters });
         onFilterSelect(newFilters);
     };
diff --git a/datahub-web-react/src/app/search/context/SearchResultContext.tsx b/datahub-web-react/src/app/search/context/SearchResultContext.tsx
index 68adead005149..961a50c1d4bfe 100644
--- a/datahub-web-react/src/app/search/context/SearchResultContext.tsx
+++ b/datahub-web-react/src/app/search/context/SearchResultContext.tsx
@@ -40,7 +40,7 @@ export const useSearchResult = () => {
 };
 
 export const useEntityType = () => {
-    return useSearchResultContext()?.searchResult.entity.type;
+    return useSearchResultContext()?.searchResult?.entity?.type;
 };
 
 export const useMatchedFields = () => {
diff --git a/datahub-web-react/src/app/search/filters/ActiveFilter.tsx b/datahub-web-react/src/app/search/filters/ActiveFilter.tsx
index 29789f872e51d..5882641cd2c9c 100644
--- a/datahub-web-react/src/app/search/filters/ActiveFilter.tsx
+++ b/datahub-web-react/src/app/search/filters/ActiveFilter.tsx
@@ -49,9 +49,18 @@ function ActiveFilter({
     const filterEntity = getFilterEntity(filterFacet.field, filterValue, availableFilters);
     const filterLabelOverride = useGetBrowseV2LabelOverride(filterFacet.field, filterValue, entityRegistry);
     const filterRenderer = useFilterRendererRegistry();
+    const facetEntity = availableFilters?.find((f) => f.field === filterFacet.field)?.entity;
 
     const { icon, label } = !filterRenderer.hasRenderer(filterFacet.field)
-        ? getFilterIconAndLabel(filterFacet.field, filterValue, entityRegistry, filterEntity, 12, filterLabelOverride)
+        ? getFilterIconAndLabel(
+              filterFacet.field,
+              filterValue,
+              entityRegistry,
+              filterEntity,
+              12,
+              filterLabelOverride,
+              facetEntity,
+          )
         : {
               icon: filterRenderer.getIcon(filterFacet.field),
               label: filterRenderer.getValueLabel(filterFacet.field, filterValue),
diff --git a/datahub-web-react/src/app/search/filters/DateRangeMenu/DateRangeMenu.tsx b/datahub-web-react/src/app/search/filters/DateRangeMenu/DateRangeMenu.tsx
new file mode 100644
index 0000000000000..aace8dcb97f48
--- /dev/null
+++ b/datahub-web-react/src/app/search/filters/DateRangeMenu/DateRangeMenu.tsx
@@ -0,0 +1,88 @@
+import moment from 'moment';
+import { Text } from '@src/alchemy-components';
+import React, { useCallback, useRef, useState } from 'react';
+import { DatePicker } from 'antd';
+import styled from 'styled-components';
+import { FacetFilterInput, FacetMetadata, FilterOperator } from '@src/types.generated';
+import { useFilterDisplayName } from '../utils';
+import useDateRangeFilterValues, { Datetime } from './useDateRangeFilterValues';
+
+const { RangePicker } = DatePicker;
+
+const Container = styled.div`
+    padding: 16px;
+    background-color: #ffffff;
+    box-shadow: 0 3px 6px -4px rgba(0, 0, 0, 0.12), 0 6px 16px 0 rgba(0, 0, 0, 0.08), 0 9px 28px 8px rgba(0, 0, 0, 0.05);
+    border-radius: 8px;
+    min-width: 225px;
+`;
+
+interface Props {
+    field: FacetMetadata;
+    manuallyUpdateFilters: (newValues: FacetFilterInput[]) => void;
+}
+
+export default function DateRangeMenu({ field, manuallyUpdateFilters }: Props) {
+    const displayName = useFilterDisplayName(field);
+    moment.tz.setDefault('GMT');
+
+    const [startDate, setStartDate] = useState<Datetime>(null);
+    const [endDate, setEndDate] = useState<Datetime>(null);
+    const [isOpen, setIsOpen] = useState<boolean>(false);
+    const ref = useRef<any>(null);
+
+    useDateRangeFilterValues({ filterField: field.field, setStartDate, setEndDate });
+
+    const handleOpenChange = useCallback(
+        (open: boolean) => {
+            setIsOpen(open);
+            if (!open) {
+                ref.current?.blur();
+                if (startDate && endDate) {
+                    manuallyUpdateFilters([
+                        {
+                            field: field.field,
+                            values: [startDate.valueOf().toString()],
+                            condition: FilterOperator.GreaterThan,
+                        },
+                        {
+                            field: field.field,
+                            values: [endDate.valueOf().toString()],
+                            condition: FilterOperator.LessThan,
+                        },
+                    ]);
+                }
+            }
+        },
+        [startDate, endDate, field.field, manuallyUpdateFilters],
+    );
+
+    const handleRangeChange = useCallback((dates: [Datetime, Datetime] | null) => {
+        const [start, end] = dates || [null, null];
+
+        start?.set({ hour: 0, minute: 0, second: 0, millisecond: 0 });
+        end?.set({ hour: 23, minute: 59, second: 59, millisecond: 999 });
+
+        setStartDate(start);
+        setEndDate(end);
+    }, []);
+
+    return (
+        <Container>
+            <Text weight="bold">Filter by {displayName}</Text>
+            <RangePicker
+                ref={ref}
+                open={isOpen}
+                allowClear
+                allowEmpty={[true, true]}
+                bordered={false}
+                value={[startDate, endDate]}
+                format="ll"
+                onChange={handleRangeChange}
+                onOpenChange={handleOpenChange}
+                onCalendarChange={() => handleOpenChange(true)}
+                style={{ paddingLeft: 0, paddingRight: 0 }}
+            />
+        </Container>
+    );
+}
diff --git a/datahub-web-react/src/app/search/filters/DateRangeMenu/useDateRangeFilterValues.ts b/datahub-web-react/src/app/search/filters/DateRangeMenu/useDateRangeFilterValues.ts
new file mode 100644
index 0000000000000..ce1810c86002f
--- /dev/null
+++ b/datahub-web-react/src/app/search/filters/DateRangeMenu/useDateRangeFilterValues.ts
@@ -0,0 +1,48 @@
+import { useEffect, useMemo } from 'react';
+import { FilterOperator } from '@src/types.generated';
+import moment from 'moment';
+import useGetSearchQueryInputs from '../../useGetSearchQueryInputs';
+
+export type Datetime = moment.Moment | null;
+
+interface Props {
+    filterField: string;
+    setStartDate: React.Dispatch<React.SetStateAction<Datetime>>;
+    setEndDate: React.Dispatch<React.SetStateAction<Datetime>>;
+}
+
+export default function useDateRangeFilterValues({ filterField, setStartDate, setEndDate }: Props) {
+    const { filters: activeFilters } = useGetSearchQueryInputs();
+    const matchingFilters = useMemo(
+        () => activeFilters.filter((f) => f.field === filterField),
+        [activeFilters, filterField],
+    );
+    const startDateFromFilter = useMemo(
+        () => matchingFilters.find((f) => f.condition === FilterOperator.GreaterThan)?.values?.[0],
+        [matchingFilters],
+    );
+    const endDateFromFilter = useMemo(
+        () => matchingFilters.find((f) => f.condition === FilterOperator.LessThan)?.values?.[0],
+        [matchingFilters],
+    );
+
+    useEffect(() => {
+        if (startDateFromFilter) {
+            setStartDate(
+                moment(parseInt(startDateFromFilter, 10)).set({ hour: 0, minute: 0, second: 0, millisecond: 0 }),
+            );
+        } else {
+            setStartDate(null);
+        }
+    }, [startDateFromFilter, setStartDate]);
+
+    useEffect(() => {
+        if (endDateFromFilter) {
+            setEndDate(
+                moment(parseInt(endDateFromFilter, 10)).set({ hour: 23, minute: 59, second: 59, millisecond: 999 }),
+            );
+        } else {
+            setEndDate(null);
+        }
+    }, [endDateFromFilter, setEndDate]);
+}
diff --git a/datahub-web-react/src/app/search/filters/EntityTypeFilter/entityTypeFilterUtils.ts b/datahub-web-react/src/app/search/filters/EntityTypeFilter/entityTypeFilterUtils.ts
index 38f8007985451..87c85a4e39dc0 100644
--- a/datahub-web-react/src/app/search/filters/EntityTypeFilter/entityTypeFilterUtils.ts
+++ b/datahub-web-react/src/app/search/filters/EntityTypeFilter/entityTypeFilterUtils.ts
@@ -91,7 +91,7 @@ export function getDisplayedFilterOptions(
                 .filter(
                     (option) => option.value.includes(FILTER_DELIMITER) && option.value.includes(filterOption.value),
                 )
-                .map((option) => ({ field: ENTITY_SUB_TYPE_FILTER_NAME, ...option }))
+                .map((option) => ({ field: ENTITY_SUB_TYPE_FILTER_NAME, ...option } as FilterOptionType))
                 .filter((o) => filterNestedOptions(o, entityRegistry, searchQuery));
             return mapFilterOption({
                 filterOption,
diff --git a/datahub-web-react/src/app/search/filters/FilterOption.tsx b/datahub-web-react/src/app/search/filters/FilterOption.tsx
index 3749f44cbf671..15851eb915e73 100644
--- a/datahub-web-react/src/app/search/filters/FilterOption.tsx
+++ b/datahub-web-react/src/app/search/filters/FilterOption.tsx
@@ -8,6 +8,7 @@ import { generateColor } from '../../entity/shared/components/styled/StyledTag';
 import { ANTD_GRAY } from '../../entity/shared/constants';
 import { useEntityRegistry } from '../../useEntityRegistry';
 import {
+    CONTAINER_FILTER_NAME,
     ENTITY_SUB_TYPE_FILTER_NAME,
     MAX_COUNT_VAL,
     PLATFORM_FILTER_NAME,
@@ -122,10 +123,11 @@ export default function FilterOption({
     addPadding,
 }: Props) {
     const [areChildrenVisible, setAreChildrenVisible] = useState(true);
-    const { field, value, count, entity } = filterOption;
+    const { field, value, count, entity, displayName } = filterOption;
     const entityRegistry = useEntityRegistry();
     const { icon, label } = getFilterIconAndLabel(field, value, entityRegistry, entity || null, 14);
-    const shouldShowIcon = field === PLATFORM_FILTER_NAME && icon !== null;
+    const finalLabel = displayName || label;
+    const shouldShowIcon = (field === PLATFORM_FILTER_NAME || field === CONTAINER_FILTER_NAME) && icon !== null;
     const shouldShowTagColor = field === TAGS_FILTER_NAME && entity?.type === EntityType.Tag;
     const isSubTypeFilter = field === TYPE_NAMES_FILTER_NAME;
     const parentEntities: Entity[] = getParentEntities(entity as Entity) || [];
@@ -155,7 +157,7 @@ export default function FilterOption({
                         nestedOptions?.map((o) => o.value),
                     )}
                     onClick={updateFilterValues}
-                    data-testid={`filter-option-${label}`}
+                    data-testid={`filter-option-${finalLabel}`}
                 >
                     {parentEntities.length > 0 && (
                         <ParentWrapper>
@@ -169,8 +171,8 @@ export default function FilterOption({
                         )}
                         {(shouldShowIcon || shouldShowTagColor) && <IconSpacer />}
                         <LabelCountWrapper>
-                            <Label ellipsis={{ tooltip: label }} style={{ maxWidth: 150 }}>
-                                {isSubTypeFilter ? capitalizeFirstLetterOnly(label as string) : label}
+                            <Label ellipsis={{ tooltip: finalLabel }} style={{ maxWidth: 150 }}>
+                                {isSubTypeFilter ? capitalizeFirstLetterOnly(finalLabel as string) : finalLabel}
                             </Label>
                             <CountText>{countText}</CountText>
                             {nestedOptions && nestedOptions.length > 0 && (
diff --git a/datahub-web-react/src/app/search/filters/MoreFilterOption.tsx b/datahub-web-react/src/app/search/filters/MoreFilterOption.tsx
index 5012f8c645d17..41df6b86c0162 100644
--- a/datahub-web-react/src/app/search/filters/MoreFilterOption.tsx
+++ b/datahub-web-react/src/app/search/filters/MoreFilterOption.tsx
@@ -1,14 +1,13 @@
 import { RightOutlined } from '@ant-design/icons';
 import { Dropdown } from 'antd';
-import styled from 'styled-components';
 import React, { useRef } from 'react';
+import styled from 'styled-components';
 import { FacetFilterInput, FacetMetadata } from '../../../types.generated';
-import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import OptionsDropdownMenu from './OptionsDropdownMenu';
-import useSearchFilterDropdown from './useSearchFilterDropdown';
 import { IconWrapper } from './SearchFilterView';
-import { getFilterDropdownIcon, useElementDimensions } from './utils';
 import { MoreFilterOptionLabel } from './styledComponents';
+import useSearchFilterDropdown from './useSearchFilterDropdown';
+import { getFilterDropdownIcon, useElementDimensions, useFilterDisplayName } from './utils';
 
 const IconNameWrapper = styled.span`
     display: flex;
@@ -34,12 +33,14 @@ export default function MoreFilterOption({ filter, activeFilters, onChangeFilter
         areFiltersLoading,
         searchQuery,
         updateSearchQuery,
+        manuallyUpdateFilters,
     } = useSearchFilterDropdown({
         filter,
         activeFilters,
         onChangeFilters,
     });
     const filterIcon = getFilterDropdownIcon(filter.field);
+    const displayName = useFilterDisplayName(filter);
 
     return (
         <Dropdown
@@ -55,7 +56,9 @@ export default function MoreFilterOption({ filter, activeFilters, onChangeFilter
                     searchQuery={searchQuery}
                     updateSearchQuery={updateSearchQuery}
                     isLoading={areFiltersLoading}
-                    searchPlaceholder={filter.displayName || ''}
+                    searchPlaceholder={displayName || ''}
+                    filter={filter}
+                    manuallyUpdateFilters={manuallyUpdateFilters}
                 />
             )}
         >
@@ -64,11 +67,11 @@ export default function MoreFilterOption({ filter, activeFilters, onChangeFilter
                 onClick={() => updateIsMenuOpen(!isMenuOpen)}
                 isActive={!!numActiveFilters}
                 isOpen={isMenuOpen}
-                data-testid={`more-filter-${capitalizeFirstLetterOnly(filter.displayName)}`}
+                data-testid={`more-filter-${displayName?.replace(/\s/g, '-')}`}
             >
                 <IconNameWrapper>
                     {filterIcon && <IconWrapper>{filterIcon}</IconWrapper>}
-                    {capitalizeFirstLetterOnly(filter.displayName)} {numActiveFilters ? `(${numActiveFilters}) ` : ''}
+                    {displayName} {numActiveFilters ? `(${numActiveFilters}) ` : ''}
                 </IconNameWrapper>
                 <RightOutlined style={{ fontSize: '12px', height: '12px' }} />
             </MoreFilterOptionLabel>
diff --git a/datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx b/datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx
index 13196af5ef85b..f1c806671a39b 100644
--- a/datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx
+++ b/datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx
@@ -2,9 +2,12 @@ import { LoadingOutlined } from '@ant-design/icons';
 import { Button } from 'antd';
 import React, { CSSProperties } from 'react';
 import styled from 'styled-components/macro';
+import { FacetFilterInput, FacetMetadata } from '@src/types.generated';
 import { useEntityRegistry } from '../../useEntityRegistry';
 import { SearchBar } from '../SearchBar';
 import { useEnterKeyListener } from '../../shared/useEnterKeyListener';
+import { getIsDateRangeFilter } from './utils';
+import DateRangeMenu from './DateRangeMenu/DateRangeMenu';
 
 const StyledButton = styled(Button)`
     width: 100%;
@@ -53,6 +56,8 @@ interface Props {
     updateSearchQuery: (query: string) => void;
     searchPlaceholder?: string;
     style?: CSSProperties;
+    filter?: FacetMetadata;
+    manuallyUpdateFilters?: (newValues: FacetFilterInput[]) => void;
 }
 
 export default function OptionsDropdownMenu({
@@ -63,11 +68,21 @@ export default function OptionsDropdownMenu({
     updateSearchQuery,
     searchPlaceholder,
     style,
+    filter,
+    manuallyUpdateFilters,
 }: Props) {
     const entityRegistry = useEntityRegistry();
 
     useEnterKeyListener({ querySelectorToExecuteClick: '#updateFiltersButton' });
 
+    if (filter && manuallyUpdateFilters && getIsDateRangeFilter(filter)) {
+        return (
+            <DropdownMenu data-testid="filter-dropdown" style={style}>
+                <DateRangeMenu field={filter} manuallyUpdateFilters={manuallyUpdateFilters} />
+            </DropdownMenu>
+        );
+    }
+
     return (
         <DropdownMenu data-testid="filter-dropdown" style={style}>
             <ScrollableContent>
diff --git a/datahub-web-react/src/app/search/filters/SearchFilter.tsx b/datahub-web-react/src/app/search/filters/SearchFilter.tsx
index 26b3a0d5ff5c2..9a69c4ec07716 100644
--- a/datahub-web-react/src/app/search/filters/SearchFilter.tsx
+++ b/datahub-web-react/src/app/search/filters/SearchFilter.tsx
@@ -1,7 +1,7 @@
 import React from 'react';
 import { FacetFilterInput, FacetMetadata } from '../../../types.generated';
 import useSearchFilterDropdown from './useSearchFilterDropdown';
-import { getFilterDropdownIcon } from './utils';
+import { getFilterDropdownIcon, useFilterDisplayName } from './utils';
 import SearchFilterView from './SearchFilterView';
 import { ENTITY_FILTER_NAME } from '../utils/constants';
 import EntityTypeFilter from './EntityTypeFilter/EntityTypeFilter';
@@ -22,12 +22,14 @@ export default function SearchFilter({ filter, activeFilters, onChangeFilters }:
         areFiltersLoading,
         searchQuery,
         updateSearchQuery,
+        manuallyUpdateFilters,
     } = useSearchFilterDropdown({
         filter,
         activeFilters,
         onChangeFilters,
     });
     const filterIcon = getFilterDropdownIcon(filter.field);
+    const displayName = useFilterDisplayName(filter);
 
     if (filter.field === ENTITY_FILTER_NAME) {
         return <EntityTypeFilter filter={filter} activeFilters={activeFilters} onChangeFilters={onChangeFilters} />;
@@ -39,12 +41,14 @@ export default function SearchFilter({ filter, activeFilters, onChangeFilters }:
             isMenuOpen={isMenuOpen}
             numActiveFilters={numActiveFilters}
             filterIcon={filterIcon}
-            displayName={filter.displayName || ''}
+            displayName={displayName || ''}
             searchQuery={searchQuery}
             loading={areFiltersLoading}
             updateIsMenuOpen={updateIsMenuOpen}
             setSearchQuery={updateSearchQuery}
             updateFilters={updateFilters}
+            filter={filter}
+            manuallyUpdateFilters={manuallyUpdateFilters}
         />
     );
 }
diff --git a/datahub-web-react/src/app/search/filters/SearchFilterView.tsx b/datahub-web-react/src/app/search/filters/SearchFilterView.tsx
index ee448e6e1acf2..c8c629d3735b8 100644
--- a/datahub-web-react/src/app/search/filters/SearchFilterView.tsx
+++ b/datahub-web-react/src/app/search/filters/SearchFilterView.tsx
@@ -2,8 +2,8 @@ import { CaretDownFilled } from '@ant-design/icons';
 import { Dropdown } from 'antd';
 import React from 'react';
 import styled from 'styled-components';
+import { FacetFilterInput, FacetMetadata } from '@src/types.generated';
 import OptionsDropdownMenu from './OptionsDropdownMenu';
-import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import { DisplayedFilterOption } from './mapFilterOption';
 import { SearchFilterLabel } from './styledComponents';
 
@@ -27,6 +27,8 @@ interface Props {
     updateIsMenuOpen: (isOpen: boolean) => void;
     setSearchQuery: (query: string) => void;
     updateFilters: () => void;
+    filter?: FacetMetadata;
+    manuallyUpdateFilters?: (newValues: FacetFilterInput[]) => void;
 }
 
 export default function SearchFilterView({
@@ -40,6 +42,8 @@ export default function SearchFilterView({
     updateIsMenuOpen,
     setSearchQuery,
     updateFilters,
+    filter,
+    manuallyUpdateFilters,
 }: Props) {
     return (
         <Dropdown
@@ -55,16 +59,18 @@ export default function SearchFilterView({
                     updateSearchQuery={setSearchQuery}
                     isLoading={loading}
                     searchPlaceholder={displayName}
+                    filter={filter}
+                    manuallyUpdateFilters={manuallyUpdateFilters}
                 />
             )}
         >
             <SearchFilterLabel
                 onClick={() => updateIsMenuOpen(!isMenuOpen)}
                 isActive={!!numActiveFilters}
-                data-testid={`filter-dropdown-${capitalizeFirstLetterOnly(displayName)}`}
+                data-testid={`filter-dropdown-${displayName?.replace(/\s/g, '-')}`}
             >
                 {filterIcon && <IconWrapper>{filterIcon}</IconWrapper>}
-                {capitalizeFirstLetterOnly(displayName)} {numActiveFilters ? `(${numActiveFilters}) ` : ''}
+                {displayName} {numActiveFilters ? `(${numActiveFilters}) ` : ''}
                 <CaretDownFilled style={{ fontSize: '12px', height: '12px' }} />
             </SearchFilterLabel>
         </Dropdown>
diff --git a/datahub-web-react/src/app/search/filters/mapFilterOption.tsx b/datahub-web-react/src/app/search/filters/mapFilterOption.tsx
index 4f2b7bf21051c..6abc3505fe3a4 100644
--- a/datahub-web-react/src/app/search/filters/mapFilterOption.tsx
+++ b/datahub-web-react/src/app/search/filters/mapFilterOption.tsx
@@ -45,7 +45,7 @@ export function mapFilterOption({
             />
         ),
         style: { padding: 0 },
-        displayName: displayName as string,
+        displayName: filterOption.displayName || (displayName as string),
         nestedOptions,
     };
 }
diff --git a/datahub-web-react/src/app/search/filters/types.ts b/datahub-web-react/src/app/search/filters/types.ts
index 3f4a705e827c4..6704c7fa01a48 100644
--- a/datahub-web-react/src/app/search/filters/types.ts
+++ b/datahub-web-react/src/app/search/filters/types.ts
@@ -5,4 +5,5 @@ export interface FilterOptionType {
     value: string;
     count?: number;
     entity?: Entity | null;
+    displayName?: string;
 }
diff --git a/datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx b/datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx
index 8467449ab6029..a614ae8e7f35e 100644
--- a/datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx
+++ b/datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx
@@ -81,6 +81,15 @@ export default function useSearchFilterDropdown({ filter, activeFilters, onChang
         setIsMenuOpen(false);
     }
 
+    function manuallyUpdateFilters(newFilters: FacetFilterInput[]) {
+        // remove any filters that are in newFilters to overwrite them
+        const filtersNotInNewFilters = activeFilters.filter(
+            (f) => !newFilters.find((newFilter) => newFilter.field === f.field),
+        );
+        onChangeFilters([...filtersNotInNewFilters, ...newFilters]);
+        setIsMenuOpen(false);
+    }
+
     function updateSearchQuery(newQuery: string) {
         setSearchQuery(newQuery);
         if (newQuery && FACETS_TO_ENTITY_TYPES[filter.field]) {
@@ -107,7 +116,13 @@ export default function useSearchFilterDropdown({ filter, activeFilters, onChang
     );
     // filter out any aggregations with a count of 0 unless it's already selected and in activeFilters
     const finalAggregations = filterEmptyAggregations(combinedAggregations, activeFilters);
-    const filterOptions = getFilterOptions(filter.field, finalAggregations, selectedFilterOptions, autoCompleteResults)
+    const filterOptions = getFilterOptions(
+        filter.field,
+        finalAggregations,
+        selectedFilterOptions,
+        autoCompleteResults,
+        filter.entity,
+    )
         .map((filterOption) =>
             mapFilterOption({ filterOption, entityRegistry, selectedFilterOptions, setSelectedFilterOptions }),
         )
@@ -122,5 +137,6 @@ export default function useSearchFilterDropdown({ filter, activeFilters, onChang
         areFiltersLoading: loading,
         searchQuery,
         updateSearchQuery,
+        manuallyUpdateFilters,
     };
 }
diff --git a/datahub-web-react/src/app/search/filters/utils.tsx b/datahub-web-react/src/app/search/filters/utils.tsx
index f115277a04967..6aea405d0465c 100644
--- a/datahub-web-react/src/app/search/filters/utils.tsx
+++ b/datahub-web-react/src/app/search/filters/utils.tsx
@@ -1,4 +1,7 @@
+import moment from 'moment-timezone';
+import { useEntityRegistry } from '@src/app/useEntityRegistry';
 import Icon from '@ant-design/icons/lib/components/Icon';
+import TableIcon from '@src/images/table-icon.svg?react';
 import {
     BookOutlined,
     DatabaseOutlined,
@@ -8,6 +11,8 @@ import {
     TagOutlined,
     UserOutlined,
 } from '@ant-design/icons';
+import { removeMarkdown } from '@src/app/entity/shared/components/styled/StripMarkdownText';
+import { DATE_TYPE_URN } from '@src/app/shared/constants';
 import React, { useLayoutEffect, useState } from 'react';
 import styled from 'styled-components';
 import {
@@ -20,6 +25,8 @@ import {
     FacetFilterInput,
     FacetMetadata,
     GlossaryTerm,
+    Container,
+    StructuredPropertyEntity,
 } from '../../../types.generated';
 import { IconStyleType } from '../../entity/Entity';
 import {
@@ -33,6 +40,7 @@ import {
     LEGACY_ENTITY_FILTER_NAME,
     OWNERS_FILTER_NAME,
     PLATFORM_FILTER_NAME,
+    STRUCTURED_PROPERTIES_FILTER_NAME,
     TAGS_FILTER_NAME,
     TYPE_NAMES_FILTER_NAME,
     UNIT_SEPARATOR,
@@ -163,6 +171,7 @@ export function getFilterIconAndLabel(
     filterEntity: Entity | null,
     size?: number,
     filterLabelOverride?: string | null,
+    facetEntity?: Entity | null,
 ) {
     let icon: React.ReactNode = null;
     let label: React.ReactNode = null;
@@ -186,6 +195,15 @@ export function getFilterIconAndLabel(
             entityRegistry.getIcon(EntityType.DataPlatform, size || 12, IconStyleType.ACCENT, ANTD_GRAY[9])
         );
         label = filterEntity ? entityRegistry.getDisplayName(EntityType.DataPlatform, filterEntity) : filterValue;
+    } else if (filterField === CONTAINER_FILTER_NAME) {
+        // Scenario where the filter entity exists and filterField is container
+        const logoUrl = (filterEntity as Container)?.platform?.properties?.logoUrl;
+        icon = logoUrl ? (
+            <PlatformIcon src={logoUrl} size={size} />
+        ) : (
+            entityRegistry.getIcon(EntityType.DataPlatform, size || 12, IconStyleType.ACCENT, ANTD_GRAY[9])
+        );
+        label = entityRegistry.getDisplayName(EntityType.Container, filterEntity);
     } else if (filterField === BROWSE_PATH_V2_FILTER_NAME) {
         icon = <FolderFilled size={size} color="black" />;
         label = getLastBrowseEntryFromFilterValue(filterValue);
@@ -196,8 +214,12 @@ export function getFilterIconAndLabel(
             filterEntity,
             size,
         );
+
         icon = newIcon;
         label = newLabel;
+    } else if (filterField.startsWith(STRUCTURED_PROPERTIES_FILTER_NAME)) {
+        label = getStructuredPropFilterDisplayName(filterField, filterValue, facetEntity);
+        icon = <Icon component={TableIcon} />;
     } else {
         label = filterValue;
     }
@@ -257,6 +279,10 @@ export function sortFacets(facetA: FacetMetadata, facetB: FacetMetadata, sortedF
 }
 
 export function getFilterDropdownIcon(field: string) {
+    if (field.startsWith(STRUCTURED_PROPERTIES_FILTER_NAME)) {
+        return <Icon component={TableIcon} />;
+    }
+
     switch (field) {
         case PLATFORM_FILTER_NAME:
             return <DatabaseOutlined />;
@@ -285,10 +311,15 @@ export function getFilterOptions(
     aggregations: AggregationMetadata[],
     selectedFilterOptions: FilterOptionType[],
     autoCompleteResults?: GetAutoCompleteMultipleResultsQuery,
+    filterEntity?: Entity | null,
 ) {
-    const aggregationFilterOptions = aggregations.map((agg) => ({ field: filterField, ...agg }));
+    const aggregationFilterOptions = aggregations.map((agg) => ({
+        field: filterField,
+        displayName: getStructuredPropFilterDisplayName(filterField, agg.value, filterEntity),
+        ...agg,
+    }));
 
-    const searchResults = autoCompleteResults?.autoCompleteForMultiple?.suggestions.find((suggestion) =>
+    const searchResults = autoCompleteResults?.autoCompleteForMultiple?.suggestions?.find((suggestion) =>
         FACETS_TO_ENTITY_TYPES[filterField]?.includes(suggestion.type),
     );
     const searchFilterOptions = searchResults?.entities
@@ -344,6 +375,9 @@ export function getParentEntities(entity: Entity): Entity[] | null {
     if (entity.type === EntityType.Domain) {
         return (entity as Domain).parentDomains?.domains || [];
     }
+    if (entity.type === EntityType.Container) {
+        return (entity as Container).parentContainers?.containers || [];
+    }
     return null;
 }
 
@@ -370,3 +404,43 @@ export function useElementDimensions(ref) {
 
     return dimensions;
 }
+
+export function getStructuredPropFilterDisplayName(field: string, value: string, entity?: Entity | null) {
+    const isStructuredPropertyValue = field.startsWith('structuredProperties.');
+    if (!isStructuredPropertyValue) return undefined;
+
+    // check for structured prop entity values
+    if (value.startsWith('urn:li:')) {
+        // this value is an urn, handle entity display names elsewhere
+        return undefined;
+    }
+
+    // check for structured prop date values
+    if (entity && (entity as StructuredPropertyEntity).definition?.valueType?.urn === DATE_TYPE_URN) {
+        return moment(parseInt(value, 10)).tz('GMT').format('MM/DD/YYYY').toString();
+    }
+
+    // check for structured prop number values
+    if (!Number.isNaN(parseFloat(value))) {
+        return parseFloat(value).toString();
+    }
+
+    return removeMarkdown(value);
+}
+
+export function useFilterDisplayName(filter: FacetMetadata, predicateDisplayName?: string) {
+    const entityRegistry = useEntityRegistry();
+
+    if (filter.entity) {
+        return entityRegistry.getDisplayName(filter.entity.type, filter.entity);
+    }
+
+    return predicateDisplayName || filter.displayName || filter.field;
+}
+
+export function getIsDateRangeFilter(field: FacetMetadata) {
+    if (field.entity && field.entity.type === EntityType.StructuredProperty) {
+        return (field.entity as StructuredPropertyEntity).definition?.valueType?.urn === DATE_TYPE_URN;
+    }
+    return false;
+}
diff --git a/datahub-web-react/src/app/search/matches/MatchedFieldList.tsx b/datahub-web-react/src/app/search/matches/MatchedFieldList.tsx
index 0bfe000dea366..9d77d446ff3b8 100644
--- a/datahub-web-react/src/app/search/matches/MatchedFieldList.tsx
+++ b/datahub-web-react/src/app/search/matches/MatchedFieldList.tsx
@@ -42,7 +42,7 @@ const RenderedField = ({
     field: MatchedField;
 }) => {
     const entityRegistry = useEntityRegistry();
-    const query = useSearchQuery()?.trim().toLowerCase();
+    const query = useSearchQuery()?.trim()?.toLowerCase();
     const customRenderedField = customFieldRenderer?.(field);
     if (customRenderedField) return <b>{customRenderedField}</b>;
     if (isHighlightableEntityField(field)) {
diff --git a/datahub-web-react/src/app/search/matches/SearchTextHighlighter.tsx b/datahub-web-react/src/app/search/matches/SearchTextHighlighter.tsx
index d8da1088ea89d..7a0a0e1e41a4b 100644
--- a/datahub-web-react/src/app/search/matches/SearchTextHighlighter.tsx
+++ b/datahub-web-react/src/app/search/matches/SearchTextHighlighter.tsx
@@ -23,7 +23,7 @@ const SearchTextHighlighter = ({ field, text, enableFullHighlight = false }: Pro
     const enableNameHighlight = appConfig.config.visualConfig.searchResult?.enableNameHighlight;
     const matchedFields = useMatchedFieldsByGroup(field);
     const hasMatchedField = !!matchedFields?.length;
-    const normalizedSearchQuery = useSearchQuery()?.trim().toLowerCase();
+    const normalizedSearchQuery = useSearchQuery()?.trim()?.toLowerCase();
     const normalizedText = text.trim().toLowerCase();
     const hasSubstring = hasMatchedField && !!normalizedSearchQuery && normalizedText.includes(normalizedSearchQuery);
     const pattern = enableFullHighlight ? HIGHLIGHT_ALL_PATTERN : undefined;
diff --git a/datahub-web-react/src/app/search/sidebar/useBrowsePagination.tsx b/datahub-web-react/src/app/search/sidebar/useBrowsePagination.tsx
index 48a71e96e4f7f..cadecdb318532 100644
--- a/datahub-web-react/src/app/search/sidebar/useBrowsePagination.tsx
+++ b/datahub-web-react/src/app/search/sidebar/useBrowsePagination.tsx
@@ -98,7 +98,7 @@ const useBrowsePagination = ({ skip }: Props) => {
         loaded: !!latestData || !!error,
         error,
         groups,
-        path: latestData?.browseV2?.metadata.path,
+        path: latestData?.browseV2?.metadata?.path,
         observable: <div ref={observableRef} style={{ width: '1px', height: '1px' }} />,
         retry,
     } as const;
diff --git a/datahub-web-react/src/app/search/utils/constants.ts b/datahub-web-react/src/app/search/utils/constants.ts
index 8616f67ac2b6a..7aa416e12c954 100644
--- a/datahub-web-react/src/app/search/utils/constants.ts
+++ b/datahub-web-react/src/app/search/utils/constants.ts
@@ -25,6 +25,11 @@ export const REMOVED_FILTER_NAME = 'removed';
 export const ORIGIN_FILTER_NAME = 'origin';
 export const DEGREE_FILTER_NAME = 'degree';
 export const BROWSE_PATH_V2_FILTER_NAME = 'browsePathV2';
+export const STRUCTURED_PROPERTIES_FILTER_NAME = 'structuredProperties.';
+export const ENTITY_TYPES_FILTER_NAME = 'entityTypes';
+export const IS_HIDDEN_PROPERTY_FILTER_NAME = 'isHidden';
+export const SHOW_IN_COLUMNS_TABLE_PROPERTY_FILTER_NAME = 'showInColumnsTable';
+export const SHOW_IN_ASSET_SUMMARY_PROPERTY_FILTER_NAME = 'showInAssetSummary';
 
 export const LEGACY_ENTITY_FILTER_FIELDS = [ENTITY_FILTER_NAME, LEGACY_ENTITY_FILTER_NAME];
 
diff --git a/datahub-web-react/src/app/search/utils/useDownloadScrollAcrossEntitiesSearchResults.ts b/datahub-web-react/src/app/search/utils/useDownloadScrollAcrossEntitiesSearchResults.ts
index b1bfb40ef0e45..dd71834b29d2a 100644
--- a/datahub-web-react/src/app/search/utils/useDownloadScrollAcrossEntitiesSearchResults.ts
+++ b/datahub-web-react/src/app/search/utils/useDownloadScrollAcrossEntitiesSearchResults.ts
@@ -25,7 +25,7 @@ export function useDownloadScrollAcrossEntitiesSearchResults(params: DownloadSea
         searchResults:
             ((data?.scrollAcrossEntities && {
                 ...data?.scrollAcrossEntities,
-                nextScrollId: data?.scrollAcrossEntities.nextScrollId,
+                nextScrollId: data?.scrollAcrossEntities?.nextScrollId,
             }) as DownloadSearchResults) || undefined,
         loading,
         error,
@@ -44,7 +44,7 @@ export function useDownloadScrollAcrossEntitiesSearchResults(params: DownloadSea
                 (res) =>
                     ((res.data?.scrollAcrossEntities && {
                         ...res.data?.scrollAcrossEntities,
-                        nextScrollId: res.data?.scrollAcrossEntities.nextScrollId,
+                        nextScrollId: res.data?.scrollAcrossEntities?.nextScrollId,
                     }) as DownloadSearchResults) || undefined,
             ),
     };
diff --git a/datahub-web-react/src/app/settings/AccessTokens.tsx b/datahub-web-react/src/app/settings/AccessTokens.tsx
index 6ccca8dd360d7..507224ed4df1f 100644
--- a/datahub-web-react/src/app/settings/AccessTokens.tsx
+++ b/datahub-web-react/src/app/settings/AccessTokens.tsx
@@ -196,8 +196,8 @@ export const AccessTokens = () => {
         return renderSearchResult(result);
     });
 
-    const totalTokens = tokensData?.listAccessTokens.total || 0;
-    const tokens = useMemo(() => tokensData?.listAccessTokens.tokens || [], [tokensData]);
+    const totalTokens = tokensData?.listAccessTokens?.total || 0;
+    const tokens = useMemo(() => tokensData?.listAccessTokens?.tokens || [], [tokensData]);
     const filteredTokens = tokens.filter((token) => !removedTokens.includes(token.id));
 
     const [revokeAccessToken, { error: revokeTokenError }] = useRevokeAccessTokenMutation();
diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx
index e3948349546ef..448782fa5d970 100644
--- a/datahub-web-react/src/app/settings/SettingsPage.tsx
+++ b/datahub-web-react/src/app/settings/SettingsPage.tsx
@@ -111,9 +111,9 @@ export const SettingsPage = () => {
     const me = useUserContext();
     const { config } = useAppConfig();
 
-    const isPoliciesEnabled = config?.policiesConfig.enabled;
-    const isIdentityManagementEnabled = config?.identityManagementConfig.enabled;
-    const isViewsEnabled = config?.viewsConfig.enabled;
+    const isPoliciesEnabled = config?.policiesConfig?.enabled;
+    const isIdentityManagementEnabled = config?.identityManagementConfig?.enabled;
+    const isViewsEnabled = config?.viewsConfig?.enabled;
     const { readOnlyModeEnabled } = config.featureFlags;
 
     const showPolicies = (isPoliciesEnabled && me && me?.platformPrivileges?.managePolicies) || false;
diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
index 8936a935735c5..64e12cf23b718 100644
--- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
+++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
@@ -8,6 +8,7 @@ import {
     SolutionOutlined,
     DownOutlined,
     GlobalOutlined,
+    UnorderedListOutlined,
 } from '@ant-design/icons';
 import { Link } from 'react-router-dom';
 import { Button, Dropdown, Tooltip } from 'antd';
@@ -15,6 +16,7 @@ import { useAppConfig, useBusinessAttributesFlag } from '../../useAppConfig';
 import { ANTD_GRAY } from '../../entity/shared/constants';
 import { HOME_PAGE_INGESTION_ID } from '../../onboarding/config/HomePageOnboardingConfig';
 import { useToggleEducationStepIdsAllowList } from '../../onboarding/useToggleEducationStepIdsAllowList';
+import { PageRoutes } from '../../../conf/Global';
 import { useUserContext } from '../../context/useUserContext';
 import DomainIcon from '../../domain/DomainIcon';
 
@@ -42,6 +44,10 @@ const NavTitleContainer = styled.span`
     padding: 2px;
 `;
 
+const NavSubItemTitleContainer = styled(NavTitleContainer)`
+    line-height: 20px;
+`;
+
 const NavTitleText = styled.span`
     margin-left: 6px;
     font-weight: bold;
@@ -64,13 +70,16 @@ export function HeaderLinks(props: Props) {
 
     const businessAttributesFlag = useBusinessAttributesFlag();
 
-    const isAnalyticsEnabled = config?.analyticsConfig.enabled;
-    const isIngestionEnabled = config?.managedIngestionConfig.enabled;
+    const isAnalyticsEnabled = config?.analyticsConfig?.enabled;
+    const isIngestionEnabled = config?.managedIngestionConfig?.enabled;
 
     const showAnalytics = (isAnalyticsEnabled && me && me?.platformPrivileges?.viewAnalytics) || false;
     const showSettings = true;
     const showIngestion =
         isIngestionEnabled && me && (me.platformPrivileges?.manageIngestion || me.platformPrivileges?.manageSecrets);
+    const showStructuredProperties =
+        config?.featureFlags?.showManageStructuredProperties &&
+        (me.platformPrivileges?.manageStructuredProperties || me.platformPrivileges?.viewStructuredPropertiesPage);
 
     useToggleEducationStepIdsAllowList(!!showIngestion, HOME_PAGE_INGESTION_ID);
 
@@ -125,6 +134,22 @@ export function HeaderLinks(props: Props) {
                   },
               ]
             : []),
+        ...(showStructuredProperties
+            ? [
+                  {
+                      key: 5,
+                      label: (
+                          <Link to={PageRoutes.STRUCTURED_PROPERTIES}>
+                              <NavSubItemTitleContainer>
+                                  <UnorderedListOutlined style={{ fontSize: '14px', fontWeight: 'bold' }} />
+                                  <NavTitleText>Structured Properties</NavTitleText>
+                              </NavSubItemTitleContainer>
+                              <NavTitleDescription>Manage custom properties for your data assets</NavTitleDescription>
+                          </Link>
+                      ),
+                  },
+              ]
+            : []),
     ];
 
     return (
diff --git a/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx b/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx
index 026cf6f6fa724..51b96b14126b3 100644
--- a/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx
+++ b/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx
@@ -155,7 +155,9 @@ export default function EditBusinessAttributeModal({
             event.stopPropagation();
         };
         /* eslint-disable-next-line react/prop-types */
-        const selectedItem = displayedAttributes.find((attribute) => attribute.urn === value)?.component;
+        const selectedItem = selectedAttribute
+            ? selectedAttribute?.props?.component
+            : displayedAttributes.find((attribute) => attribute.urn === value)?.component;
         return (
             <StyleTag onMouseDown={onPreventMouseDown} closable={closable} onClose={onClose}>
                 {selectedItem}
@@ -172,13 +174,13 @@ export default function EditBusinessAttributeModal({
     const onSelectValue = (selectedUrn: string) => {
         const selectedSearchOption = attributeSearchOptions?.find((option) => option.props.value === selectedUrn);
         setUrn(selectedUrn);
-        if (!selectedAttribute) {
+        if (selectedAttribute?.selectedUrn !== selectedUrn) {
             setSelectedAttribute({
                 selectedUrn,
                 component: (
                     <div>
                         <GlobalOutlined />
-                        <AttributeName>{selectedSearchOption?.props.name}</AttributeName>
+                        <AttributeName>{selectedSearchOption?.props?.name}</AttributeName>
                     </div>
                 ),
             });
diff --git a/datahub-web-react/src/app/shared/constants.ts b/datahub-web-react/src/app/shared/constants.ts
index 1cd9077ab8cdf..5ccc6e5698faa 100644
--- a/datahub-web-react/src/app/shared/constants.ts
+++ b/datahub-web-react/src/app/shared/constants.ts
@@ -1,5 +1,8 @@
 export const ENTER_KEY_CODE = 13;
 
+export const DATE_TYPE_URN = 'urn:li:dataType:datahub.date';
+export const TYPE_ICON_CLASS_NAME = 'typeIcon';
+
 export enum ErrorCodes {
     BadRequest = 400,
     Unauthorized = 401,
diff --git a/datahub-web-react/src/app/shared/sidebar/EmptySectionText.tsx b/datahub-web-react/src/app/shared/sidebar/EmptySectionText.tsx
new file mode 100644
index 0000000000000..10236261fd577
--- /dev/null
+++ b/datahub-web-react/src/app/shared/sidebar/EmptySectionText.tsx
@@ -0,0 +1,20 @@
+import React from 'react';
+import styled from 'styled-components';
+import { Typography } from 'antd';
+
+const EmptyContentMessage = styled(Typography.Text)`
+    font-size: 12px;
+    font-weight: 400;
+    color: #56668e;
+    opacity: 0.5;
+`;
+
+type Props = {
+    message: string;
+};
+
+const EmptySectionText = ({ message }: Props) => {
+    return <EmptyContentMessage type="secondary">{message}.</EmptyContentMessage>;
+};
+
+export default EmptySectionText;
diff --git a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx
index 7f2f80e023a6b..cd233552e68aa 100644
--- a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx
+++ b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx
@@ -236,7 +236,7 @@ export default function EditTagTermsModal({
         setUrns(newUrns);
         setSelectedTerms([
             ...selectedTerms,
-            { urn, component: <TagTermLabel termName={selectedSearchOption?.props.name} /> },
+            { urn, component: <TagTermLabel termName={selectedSearchOption?.props?.name} /> },
         ]);
         setSelectedTags([
             ...selectedTags,
diff --git a/datahub-web-react/src/app/sharedV2/modals/ConfirmationModal.tsx b/datahub-web-react/src/app/sharedV2/modals/ConfirmationModal.tsx
new file mode 100644
index 0000000000000..34c317d27a0a4
--- /dev/null
+++ b/datahub-web-react/src/app/sharedV2/modals/ConfirmationModal.tsx
@@ -0,0 +1,76 @@
+import { Button, Heading, Text, typography } from '@components';
+import { Modal } from 'antd';
+import React from 'react';
+import styled from 'styled-components';
+
+const ButtonsContainer = styled.div`
+    display: flex;
+    gap: 16px;
+    justify-content: end;
+`;
+
+export const StyledModal = styled(Modal)`
+    font-family: ${typography.fonts.body};
+
+    &&& .ant-modal-content {
+        box-shadow: 0px 4px 12px 0px rgba(9, 1, 61, 0.12);
+        border-radius: 12px;
+    }
+
+    .ant-modal-header {
+        border-bottom: 0;
+        padding-bottom: 0;
+        border-radius: 12px !important;
+    }
+
+    .ant-modal-body {
+        padding: 12px 24px;
+    }
+`;
+
+interface Props {
+    isOpen: boolean;
+    handleConfirm: (e: any) => void;
+    handleClose: () => void;
+    modalTitle?: string;
+    modalText?: string | React.ReactNode;
+    closeButtonText?: string;
+    confirmButtonText?: string;
+}
+
+export const ConfirmationModal = ({
+    isOpen,
+    handleClose,
+    handleConfirm,
+    modalTitle,
+    modalText,
+    closeButtonText,
+    confirmButtonText,
+}: Props) => {
+    return (
+        <StyledModal
+            open={isOpen}
+            onCancel={handleClose}
+            centered
+            footer={
+                <ButtonsContainer>
+                    <Button variant="text" onClick={handleClose} data-testid="modal-cancel-button">
+                        {closeButtonText || 'Cancel'}
+                    </Button>
+                    <Button variant="filled" onClick={handleConfirm} data-testid="modal-confirm-button">
+                        {confirmButtonText || 'Yes'}
+                    </Button>
+                </ButtonsContainer>
+            }
+            title={
+                <Heading type="h1" weight="bold">
+                    {modalTitle || 'Confirm'}
+                </Heading>
+            }
+        >
+            <Text color="gray" size="lg">
+                {modalText || 'Are you sure?'}
+            </Text>
+        </StyledModal>
+    );
+};
diff --git a/datahub-web-react/src/app/sharedV2/toastMessageUtils.ts b/datahub-web-react/src/app/sharedV2/toastMessageUtils.ts
new file mode 100644
index 0000000000000..b714d668e05fc
--- /dev/null
+++ b/datahub-web-react/src/app/sharedV2/toastMessageUtils.ts
@@ -0,0 +1,18 @@
+import { message } from 'antd';
+import { ReactNode } from 'react';
+
+export enum ToastType {
+    INFO = 'info',
+    ERROR = 'error',
+    SUCCESS = 'success',
+    WARNING = 'warning',
+    LOADING = 'loading',
+}
+
+export const showToastMessage = (type: ToastType, content: string | ReactNode, duration: number) => {
+    message.open({
+        type,
+        content,
+        duration,
+    });
+};
diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx
index f119f8bc7d53a..d7fef85db4b62 100644
--- a/datahub-web-react/src/appConfigContext.tsx
+++ b/datahub-web-react/src/appConfigContext.tsx
@@ -56,6 +56,7 @@ export const DEFAULT_APP_CONFIG = {
         dataContractsEnabled: false,
         editableDatasetNameEnabled: false,
         showSeparateSiblings: false,
+        showManageStructuredProperties: false,
     },
 };
 
diff --git a/datahub-web-react/src/conf/Global.ts b/datahub-web-react/src/conf/Global.ts
index e0172d3f07e41..7b3925f8c6ba1 100644
--- a/datahub-web-react/src/conf/Global.ts
+++ b/datahub-web-react/src/conf/Global.ts
@@ -25,6 +25,7 @@ export enum PageRoutes {
     DOMAINS = '/domains',
     DOMAIN = '/domain',
     GLOSSARY = '/glossary',
+    STRUCTURED_PROPERTIES = '/structured-properties',
     SETTINGS_VIEWS = '/settings/views',
     EMBED = '/embed',
     EMBED_LOOKUP = '/embed/lookup/:url',
diff --git a/datahub-web-react/src/fonts/Mulish-Black.ttf b/datahub-web-react/src/fonts/Mulish-Black.ttf
new file mode 100644
index 0000000000000..7c45457b315e1
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-Black.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-BlackItalic.ttf b/datahub-web-react/src/fonts/Mulish-BlackItalic.ttf
new file mode 100644
index 0000000000000..7eda815bc44e3
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-BlackItalic.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-Bold.ttf b/datahub-web-react/src/fonts/Mulish-Bold.ttf
new file mode 100644
index 0000000000000..0c1eed735a9a8
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-Bold.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-BoldItalic.ttf b/datahub-web-react/src/fonts/Mulish-BoldItalic.ttf
new file mode 100644
index 0000000000000..6f08e1512b788
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-BoldItalic.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-ExtraBold.ttf b/datahub-web-react/src/fonts/Mulish-ExtraBold.ttf
new file mode 100644
index 0000000000000..bf116994f6797
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-ExtraBold.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-ExtraBoldItalic.ttf b/datahub-web-react/src/fonts/Mulish-ExtraBoldItalic.ttf
new file mode 100644
index 0000000000000..33ae716cefaf1
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-ExtraBoldItalic.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-ExtraLight.ttf b/datahub-web-react/src/fonts/Mulish-ExtraLight.ttf
new file mode 100644
index 0000000000000..7964001697e4b
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-ExtraLight.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-ExtraLightItalic.ttf b/datahub-web-react/src/fonts/Mulish-ExtraLightItalic.ttf
new file mode 100644
index 0000000000000..3e57b5451caa5
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-ExtraLightItalic.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-Italic.ttf b/datahub-web-react/src/fonts/Mulish-Italic.ttf
new file mode 100644
index 0000000000000..a42d8759e0b48
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-Italic.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-Light.ttf b/datahub-web-react/src/fonts/Mulish-Light.ttf
new file mode 100644
index 0000000000000..f4d91c3b92f6a
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-Light.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-LightItalic.ttf b/datahub-web-react/src/fonts/Mulish-LightItalic.ttf
new file mode 100644
index 0000000000000..a4712b16769c0
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-LightItalic.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-Medium.ttf b/datahub-web-react/src/fonts/Mulish-Medium.ttf
new file mode 100644
index 0000000000000..be50c59750cdf
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-Medium.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-MediumItalic.ttf b/datahub-web-react/src/fonts/Mulish-MediumItalic.ttf
new file mode 100644
index 0000000000000..01cf746fd2db0
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-MediumItalic.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-Regular.ttf b/datahub-web-react/src/fonts/Mulish-Regular.ttf
new file mode 100644
index 0000000000000..0971518b2d645
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-Regular.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-SemiBold.ttf b/datahub-web-react/src/fonts/Mulish-SemiBold.ttf
new file mode 100644
index 0000000000000..9ac1fc8f29d59
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-SemiBold.ttf differ
diff --git a/datahub-web-react/src/fonts/Mulish-SemiBoldItalic.ttf b/datahub-web-react/src/fonts/Mulish-SemiBoldItalic.ttf
new file mode 100644
index 0000000000000..71b3dc84be2f7
Binary files /dev/null and b/datahub-web-react/src/fonts/Mulish-SemiBoldItalic.ttf differ
diff --git a/datahub-web-react/src/graphql-mock/mutationHelper.ts b/datahub-web-react/src/graphql-mock/mutationHelper.ts
index a97b41b53bc65..0cf4f5f87f29c 100644
--- a/datahub-web-react/src/graphql-mock/mutationHelper.ts
+++ b/datahub-web-react/src/graphql-mock/mutationHelper.ts
@@ -99,6 +99,7 @@ export const updateEntityLink = ({ entity, institutionalMemory }: UpdateEntityLi
             description: e.description as string,
             label: e.description as string,
             author: { urn: e.author, username: '', type: EntityType.CorpUser },
+            actor: { urn: e.author, username: '', type: EntityType.CorpUser },
             created: { time: Date.now(), actor: getActor(), __typename: 'AuditStamp' },
             associatedUrn: dataEntity.urn,
         };
diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql
index 2a0baf238761f..0d1999f82f77c 100644
--- a/datahub-web-react/src/graphql/app.graphql
+++ b/datahub-web-react/src/graphql/app.graphql
@@ -71,6 +71,7 @@ query appConfig {
             dataContractsEnabled
             editableDatasetNameEnabled
             showSeparateSiblings
+            showManageStructuredProperties
         }
     }
 }
diff --git a/datahub-web-react/src/graphql/container.graphql b/datahub-web-react/src/graphql/container.graphql
index 4b3ecfe8aaaff..88ed7582e62c5 100644
--- a/datahub-web-react/src/graphql/container.graphql
+++ b/datahub-web-react/src/graphql/container.graphql
@@ -59,6 +59,13 @@ query getContainer($urn: String!) {
         status {
             removed
         }
+        access {
+            roles {
+                role {
+                    urn
+                }
+            }
+        }
         autoRenderAspects: aspects(input: { autoRenderOnly: true }) {
             ...autoRenderAspectFields
         }
diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql
index 3897a2ced85b8..2e96a78b0f44b 100644
--- a/datahub-web-react/src/graphql/domain.graphql
+++ b/datahub-web-react/src/graphql/domain.graphql
@@ -19,9 +19,8 @@ query getDomain($urn: String!) {
         institutionalMemory {
             elements {
                 url
-                author {
-                    urn
-                    username
+                actor {
+                    ...resolvedActorFields
                 }
                 description
                 created {
diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql
index 7ce4082c42f61..788c68349b426 100644
--- a/datahub-web-react/src/graphql/fragments.graphql
+++ b/datahub-web-react/src/graphql/fragments.graphql
@@ -202,12 +202,22 @@ fragment embedFields on Embed {
     renderUrl
 }
 
+fragment resolvedActorFields on ResolvedActor {
+    ... on CorpUser {
+        urn
+        ...entityDisplayNameFields
+    }
+    ... on CorpGroup {
+        urn
+        ...entityDisplayNameFields
+    }
+}
+
 fragment institutionalMemoryFields on InstitutionalMemory {
     elements {
         url
-        author {
-            urn
-            username
+        actor {
+            ...resolvedActorFields
         }
         description
         created {
@@ -579,6 +589,11 @@ fragment nonRecursiveMLFeature on MLFeature {
             }
         }
     }
+    structuredProperties {
+        properties {
+            ...structuredPropertiesFields
+        }
+    }
 }
 
 fragment nonRecursiveMLPrimaryKey on MLPrimaryKey {
@@ -649,6 +664,11 @@ fragment nonRecursiveMLPrimaryKey on MLPrimaryKey {
             }
         }
     }
+    structuredProperties {
+        properties {
+            ...structuredPropertiesFields
+        }
+    }
 }
 
 fragment nonRecursiveMLFeatureTable on MLFeatureTable {
@@ -705,6 +725,11 @@ fragment nonRecursiveMLFeatureTable on MLFeatureTable {
     browsePathV2 {
         ...browsePathV2Fields
     }
+    structuredProperties {
+        properties {
+            ...structuredPropertiesFields
+        }
+    }
 }
 
 fragment schemaFieldFields on SchemaField {
@@ -1000,6 +1025,7 @@ fragment entityContainer on Container {
 
 fragment parentContainerFields on Container {
     urn
+    type
     properties {
         name
     }
@@ -1214,6 +1240,12 @@ fragment entityDisplayNameFields on Entity {
     ... on DataPlatformInstance {
         instanceId
     }
+    ... on StructuredPropertyEntity {
+        definition {
+            displayName
+            qualifiedName
+        }
+    }
 }
 
 fragment ermodelrelationPropertiesFields on ERModelRelationshipProperties {
@@ -1267,12 +1299,16 @@ fragment structuredPropertyFields on StructuredPropertyEntity {
         cardinality
         immutable
         valueType {
+            urn
+            type
             info {
                 type
                 displayName
             }
         }
         entityTypes {
+            urn
+            type
             info {
                 type
             }
@@ -1299,6 +1335,35 @@ fragment structuredPropertyFields on StructuredPropertyEntity {
             }
             description
         }
+        created {
+            time
+            actor {
+                urn
+                editableProperties {
+                    displayName
+                    pictureLink
+                }
+                ...entityDisplayNameFields
+            }
+        }
+        lastModified {
+            time
+            actor {
+                urn
+                editableProperties {
+                    displayName
+                    pictureLink
+                }
+                ...entityDisplayNameFields
+            }
+        }
+    }
+    settings {
+        isHidden
+        showInSearchFilters
+        showAsAssetBadge
+        showInAssetSummary
+        showInColumnsTable
     }
 }
 
@@ -1320,6 +1385,7 @@ fragment structuredPropertiesFields on StructuredPropertiesEntry {
         type
         ...entityDisplayNameFields
     }
+    associatedUrn
 }
 
 fragment autoRenderAspectFields on RawAspect {
diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql
index c4936fc3c8abe..ee05811cbb72d 100644
--- a/datahub-web-react/src/graphql/lineage.graphql
+++ b/datahub-web-react/src/graphql/lineage.graphql
@@ -62,6 +62,11 @@ fragment lineageNodeProperties on EntityWithRelationships {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on DataFlow {
         orchestrator
@@ -97,6 +102,11 @@ fragment lineageNodeProperties on EntityWithRelationships {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on Dashboard {
         urn
@@ -155,6 +165,11 @@ fragment lineageNodeProperties on EntityWithRelationships {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on Chart {
         tool
@@ -188,6 +203,11 @@ fragment lineageNodeProperties on EntityWithRelationships {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on Dataset {
         name
@@ -227,6 +247,11 @@ fragment lineageNodeProperties on EntityWithRelationships {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on MLModelGroup {
         urn
@@ -243,6 +268,11 @@ fragment lineageNodeProperties on EntityWithRelationships {
         status {
             removed
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on MLModel {
         urn
@@ -259,6 +289,11 @@ fragment lineageNodeProperties on EntityWithRelationships {
         status {
             removed
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on MLFeatureTable {
         ...nonRecursiveMLFeatureTable
diff --git a/datahub-web-react/src/graphql/me.graphql b/datahub-web-react/src/graphql/me.graphql
index 549aae1ad17de..e02a480fffe76 100644
--- a/datahub-web-react/src/graphql/me.graphql
+++ b/datahub-web-react/src/graphql/me.graphql
@@ -51,6 +51,8 @@ query getMe {
             manageGlobalAnnouncements
             createBusinessAttributes
             manageBusinessAttributes
+            manageStructuredProperties
+            viewStructuredPropertiesPage
         }
     }
 }
diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql
index 38c9df0a636d0..58c9a51f3d7e9 100644
--- a/datahub-web-react/src/graphql/search.graphql
+++ b/datahub-web-react/src/graphql/search.graphql
@@ -396,9 +396,19 @@ fragment searchResultsWithoutSchemaField on Entity {
                 type
                 ... on Dataset {
                     ...nonSiblingsDatasetSearchFields
+                    structuredProperties {
+                        properties {
+                            ...structuredPropertiesFields
+                        }
+                    }
                 }
             }
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on Role {
         id
@@ -509,6 +519,11 @@ fragment searchResultsWithoutSchemaField on Entity {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on Chart {
         chartId
@@ -578,6 +593,11 @@ fragment searchResultsWithoutSchemaField on Entity {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on DataFlow {
         flowId
@@ -619,6 +639,11 @@ fragment searchResultsWithoutSchemaField on Entity {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on DataJob {
         dataFlow {
@@ -671,6 +696,11 @@ fragment searchResultsWithoutSchemaField on Entity {
         health {
             ...entityHealth
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on GlossaryTerm {
         name
@@ -696,12 +726,22 @@ fragment searchResultsWithoutSchemaField on Entity {
         domain {
             ...entityDomain
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on GlossaryNode {
         ...glossaryNode
         parentNodes {
             ...parentNodesFields
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on Domain {
         properties {
@@ -715,6 +755,11 @@ fragment searchResultsWithoutSchemaField on Entity {
             ...parentDomainsFields
         }
         ...domainEntitiesFields
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on Container {
         properties {
@@ -752,6 +797,11 @@ fragment searchResultsWithoutSchemaField on Entity {
         parentContainers {
             ...parentContainersFields
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on MLFeatureTable {
         name
@@ -777,12 +827,27 @@ fragment searchResultsWithoutSchemaField on Entity {
         dataPlatformInstance {
             ...dataPlatformInstanceFields
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on MLFeature {
         ...nonRecursiveMLFeature
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on MLPrimaryKey {
         ...nonRecursiveMLPrimaryKey
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on MLModel {
         name
@@ -800,6 +865,11 @@ fragment searchResultsWithoutSchemaField on Entity {
         dataPlatformInstance {
             ...dataPlatformInstanceFields
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on MLModelGroup {
         name
@@ -817,6 +887,11 @@ fragment searchResultsWithoutSchemaField on Entity {
         dataPlatformInstance {
             ...dataPlatformInstanceFields
         }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
     }
     ... on Tag {
         name
@@ -855,6 +930,9 @@ fragment searchResultsWithoutSchemaField on Entity {
     ... on BusinessAttribute {
         ...businessAttributeFields
     }
+    ... on StructuredPropertyEntity {
+        ...structuredPropertyFields
+    }
 }
 
 fragment searchResultFields on Entity {
@@ -867,12 +945,25 @@ fragment searchResultFields on Entity {
 fragment facetFields on FacetMetadata {
     field
     displayName
+    entity {
+        urn
+        type
+        ...entityDisplayNameFields
+        ... on StructuredPropertyEntity {
+            definition {
+                valueType {
+                    urn
+                }
+            }
+        }
+    }
     aggregations {
         value
         count
         entity {
             urn
             type
+            ...entityDisplayNameFields
             ... on Tag {
                 name
                 properties {
@@ -910,6 +1001,9 @@ fragment facetFields on FacetMetadata {
                 properties {
                     name
                 }
+                parentContainers {
+                    ...parentContainersFields
+                }
             }
             ... on CorpUser {
                 username
@@ -980,6 +1074,12 @@ fragment entityField on SchemaFieldEntity {
             platform {
                 ...platformFields
             }
+            dataPlatformInstance {
+                ...dataPlatformInstanceFields
+            }
+            parentContainers {
+                ...parentContainersFields
+            }
         }
     }
     fieldPath
diff --git a/datahub-web-react/src/graphql/structuredProperties.graphql b/datahub-web-react/src/graphql/structuredProperties.graphql
index 60079db3b1cd4..8a59b6e5fe0a0 100644
--- a/datahub-web-react/src/graphql/structuredProperties.graphql
+++ b/datahub-web-react/src/graphql/structuredProperties.graphql
@@ -1,3 +1,116 @@
+query getStructuredProperty($urn: String!) {
+    entity(urn: $urn) {
+        urn
+        type
+        ... on StructuredPropertyEntity {
+            urn
+            type
+            definition {
+                qualifiedName
+                displayName
+                description
+                cardinality
+                entityTypes {
+                    urn
+                    type
+                    info {
+                        type
+                    }
+                }
+                valueType {
+                    urn
+                    type
+                    info {
+                        type
+                        qualifiedName
+                        displayName
+                        description
+                    }
+                }
+                allowedValues {
+                    value {
+                        ... on StringValue {
+                            stringValue
+                        }
+                        ... on NumberValue {
+                            numberValue
+                        }
+                    }
+                    description
+                }
+                typeQualifier {
+                    allowedTypes {
+                        urn
+                        type
+                        info {
+                            type
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+query searchStructuredProperties($query: String!, $start: Int!, $count: Int!) {
+    searchAcrossEntities(input: { query: $query, types: [STRUCTURED_PROPERTY], start: $start, count: $count }) {
+        start
+        count
+        total
+        searchResults {
+            entity {
+                ... on StructuredPropertyEntity {
+                    urn
+                    type
+                    definition {
+                        qualifiedName
+                        displayName
+                        description
+                        cardinality
+                        entityTypes {
+                            urn
+                            type
+                            info {
+                                type
+                            }
+                        }
+                        valueType {
+                            urn
+                            type
+                            info {
+                                type
+                                qualifiedName
+                                displayName
+                                description
+                            }
+                        }
+                        allowedValues {
+                            value {
+                                ... on StringValue {
+                                    stringValue
+                                }
+                                ... on NumberValue {
+                                    numberValue
+                                }
+                            }
+                            description
+                        }
+                        typeQualifier {
+                            allowedTypes {
+                                urn
+                                type
+                                info {
+                                    type
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
 mutation upsertStructuredProperties($input: UpsertStructuredPropertiesInput!) {
     upsertStructuredProperties(input: $input) {
         properties {
@@ -5,3 +118,27 @@ mutation upsertStructuredProperties($input: UpsertStructuredPropertiesInput!) {
         }
     }
 }
+
+mutation removeStructuredProperties($input: RemoveStructuredPropertiesInput!) {
+    removeStructuredProperties(input: $input) {
+        properties {
+            ...structuredPropertiesFields
+        }
+    }
+}
+
+mutation createStructuredProperty($input: CreateStructuredPropertyInput!) {
+    createStructuredProperty(input: $input) {
+        ...structuredPropertyFields
+    }
+}
+
+mutation updateStructuredProperty($input: UpdateStructuredPropertyInput!) {
+    updateStructuredProperty(input: $input) {
+        ...structuredPropertyFields
+    }
+}
+
+mutation deleteStructuredProperty($input: DeleteStructuredPropertyInput!) {
+    deleteStructuredProperty(input: $input)
+}
diff --git a/datahub-web-react/src/images/cassandralogo.png b/datahub-web-react/src/images/cassandralogo.png
new file mode 100644
index 0000000000000..e497dbb6ee76b
Binary files /dev/null and b/datahub-web-react/src/images/cassandralogo.png differ
diff --git a/datahub-web-react/src/images/dremiologo.png b/datahub-web-react/src/images/dremiologo.png
new file mode 100644
index 0000000000000..95503c049c026
Binary files /dev/null and b/datahub-web-react/src/images/dremiologo.png differ
diff --git a/datahub-web-react/src/images/empty-forms.svg b/datahub-web-react/src/images/empty-forms.svg
new file mode 100644
index 0000000000000..e803bd4ccee2b
--- /dev/null
+++ b/datahub-web-react/src/images/empty-forms.svg
@@ -0,0 +1,48 @@
+<svg width="200" height="200" viewBox="0 0 200 200" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect width="200" height="200" rx="100" fill="#F8F7FD"/>
+<g filter="url(#filter0_d_1052_31985)">
+<path d="M66.4862 157.106H133.513C142.171 157.106 149.189 150.088 149.189 141.43V72.8559C149.189 68.6984 147.537 64.7113 144.598 61.7715L128.645 45.819C125.705 42.8792 121.718 41.2277 117.561 41.2277H66.4862C57.8288 41.2277 50.8106 48.2459 50.8106 56.9033V141.43C50.8106 150.088 57.8288 157.106 66.4862 157.106Z" fill="url(#paint0_linear_1052_31985)" shape-rendering="crispEdges"/>
+<path d="M133.513 156.971H66.4862C57.9034 156.971 50.9457 150.013 50.9457 141.43V56.9033C50.9457 48.3205 57.9034 41.3628 66.4862 41.3628H117.561C121.682 41.3628 125.635 43.0001 128.55 45.9145L144.502 61.867C147.416 64.7815 149.054 68.7343 149.054 72.8559V141.43C149.054 150.013 142.096 156.971 133.513 156.971Z" stroke="url(#paint1_linear_1052_31985)" stroke-width="0.27027" shape-rendering="crispEdges"/>
+</g>
+<g filter="url(#filter1_d_1052_31985)">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M137.568 69.3356C128.91 69.3356 121.892 62.3174 121.892 53.6599V41.8379C124.421 42.5652 126.751 43.9246 128.645 45.819L144.598 61.7715C146.693 63.8667 148.134 66.4939 148.789 69.3356H137.568Z" fill="url(#paint2_linear_1052_31985)" shape-rendering="crispEdges"/>
+</g>
+<defs>
+<filter id="filter0_d_1052_31985" x="46.7565" y="37.1736" width="106.486" height="123.987" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
+<feOffset/>
+<feGaussianBlur stdDeviation="2.02703"/>
+<feComposite in2="hardAlpha" operator="out"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.25 0"/>
+<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_1052_31985"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_1052_31985" result="shape"/>
+</filter>
+<filter id="filter1_d_1052_31985" x="117.838" y="37.7838" width="35.0048" height="35.6058" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
+<feOffset/>
+<feGaussianBlur stdDeviation="2.02703"/>
+<feComposite in2="hardAlpha" operator="out"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.12 0"/>
+<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_1052_31985"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_1052_31985" result="shape"/>
+</filter>
+<linearGradient id="paint0_linear_1052_31985" x1="99.9997" y1="41.2277" x2="99.9997" y2="139.606" gradientUnits="userSpaceOnUse">
+<stop stop-color="white"/>
+<stop offset="1" stop-color="white" stop-opacity="0.5"/>
+</linearGradient>
+<linearGradient id="paint1_linear_1052_31985" x1="149.189" y1="41.2277" x2="50.8105" y2="139.606" gradientUnits="userSpaceOnUse">
+<stop stop-color="#F6FBFF" stop-opacity="0.5"/>
+<stop offset="0.333333" stop-color="white" stop-opacity="0"/>
+<stop offset="0.666667" stop-color="#F9FCFF" stop-opacity="0"/>
+<stop offset="1" stop-color="white" stop-opacity="0.7"/>
+</linearGradient>
+<linearGradient id="paint2_linear_1052_31985" x1="121.892" y1="69.3355" x2="220.269" y2="-29.0417" gradientUnits="userSpaceOnUse">
+<stop stop-color="#F6FBFF" stop-opacity="0.5"/>
+<stop offset="0.333333" stop-color="white" stop-opacity="0"/>
+<stop offset="0.666667" stop-color="#F9FCFF" stop-opacity="0"/>
+<stop offset="1" stop-color="white" stop-opacity="0.7"/>
+</linearGradient>
+</defs>
+</svg>
diff --git a/datahub-web-react/src/images/hudilogo.png b/datahub-web-react/src/images/hudilogo.png
new file mode 100644
index 0000000000000..4b58cc5a34826
Binary files /dev/null and b/datahub-web-react/src/images/hudilogo.png differ
diff --git a/datahub-web-react/src/images/neo4j.png b/datahub-web-react/src/images/neo4j.png
new file mode 100644
index 0000000000000..b03b2a4532b3b
Binary files /dev/null and b/datahub-web-react/src/images/neo4j.png differ
diff --git a/datahub-web-react/src/images/table-icon.svg b/datahub-web-react/src/images/table-icon.svg
new file mode 100644
index 0000000000000..e9535061ab87e
--- /dev/null
+++ b/datahub-web-react/src/images/table-icon.svg
@@ -0,0 +1,3 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M17.5 3.75H2.5C2.33424 3.75 2.17527 3.81585 2.05806 3.93306C1.94085 4.05027 1.875 4.20924 1.875 4.375V15C1.875 15.3315 2.0067 15.6495 2.24112 15.8839C2.47554 16.1183 2.79348 16.25 3.125 16.25H16.875C17.2065 16.25 17.5245 16.1183 17.7589 15.8839C17.9933 15.6495 18.125 15.3315 18.125 15V4.375C18.125 4.20924 18.0592 4.05027 17.9419 3.93306C17.8247 3.81585 17.6658 3.75 17.5 3.75ZM3.125 8.75H6.25V11.25H3.125V8.75ZM7.5 8.75H16.875V11.25H7.5V8.75ZM16.875 5V7.5H3.125V5H16.875ZM3.125 12.5H6.25V15H3.125V12.5ZM16.875 15H7.5V12.5H16.875V15Z" fill="currentColor"/>
+</svg>
diff --git a/datahub-web-react/src/providers/EducationStepsProvider.tsx b/datahub-web-react/src/providers/EducationStepsProvider.tsx
index f254b21ea99a5..807d15322227b 100644
--- a/datahub-web-react/src/providers/EducationStepsProvider.tsx
+++ b/datahub-web-react/src/providers/EducationStepsProvider.tsx
@@ -9,7 +9,7 @@ export function EducationStepsProvider({ children }: { children: React.ReactNode
     const userUrn = useUserContext()?.user?.urn;
     const stepIds = getStepIds(userUrn || '');
     const { data } = useBatchGetStepStatesQuery({ skip: !userUrn, variables: { input: { ids: stepIds } } });
-    const results = data?.batchGetStepStates.results;
+    const results = data?.batchGetStepStates?.results;
     const [educationSteps, setEducationSteps] = useState<StepStateResult[] | null>(results || null);
     const [educationStepIdsAllowlist, setEducationStepIdsAllowlist] = useState<Set<string>>(
         new Set(getInitialAllowListIds()),
diff --git a/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx b/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx
index eaca969bb524e..f03fc3b43f320 100644
--- a/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx
+++ b/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx
@@ -26,6 +26,7 @@ import { DataPlatformEntity } from '../../app/entity/dataPlatform/DataPlatformEn
 import { ContainerEntity } from '../../app/entity/container/ContainerEntity';
 import AppConfigProvider from '../../AppConfigProvider';
 import { BusinessAttributeEntity } from '../../app/entity/businessAttribute/BusinessAttributeEntity';
+import { SchemaFieldPropertiesEntity } from '../../app/entity/schemaField/SchemaFieldPropertiesEntity';
 
 type Props = {
     children: React.ReactNode;
@@ -49,6 +50,7 @@ export function getTestEntityRegistry() {
     entityRegistry.register(new DataPlatformEntity());
     entityRegistry.register(new ContainerEntity());
     entityRegistry.register(new BusinessAttributeEntity());
+    entityRegistry.register(new SchemaFieldPropertiesEntity());
     return entityRegistry;
 }
 
diff --git a/datahub-web-react/tsconfig.json b/datahub-web-react/tsconfig.json
index 56361d52b21c3..ebdff3429acd2 100644
--- a/datahub-web-react/tsconfig.json
+++ b/datahub-web-react/tsconfig.json
@@ -1,5 +1,6 @@
 {
     "compilerOptions": {
+        "baseUrl": ".",
         "target": "es2017",
         "lib": ["dom", "dom.iterable", "esnext"],
         "allowJs": true,
@@ -13,10 +14,33 @@
         "module": "esnext",
         "moduleResolution": "node",
         "resolveJsonModule": true,
-        "isolatedModules": true,
+        "isolatedModules": false,
         "noEmit": true,
         "jsx": "react-jsx",
-        "types": ["vitest/globals"]
+        "types": ["vitest/globals"],
+        "paths": {
+            "@src/*": ["./src/*"],
+            "@app/*": ["./src/app/*"],
+            "@conf/*": ["./src/conf/*"],
+            "@components": ["./src/alchemy-components"],
+            "@components/*": ["./src/alchemy-components/*"],
+            "@graphql/*": ["./src/graphql/*"],
+            "@graphql-mock/*": ["./src/graphql-mock/*"],
+            "@images/*": ["./src/images/*"],
+            "@providers/*": ["./src/providers/*"],
+            "@utils/*": ["./src/utils/*"],
+            "@app/entityV1/*": ["./src/app/entity/*"],
+            "@app/entityV2/*": ["./src/app/entityV2/*"],
+            "@app/searchV2/*": ["./src/app/searchV2/*"],
+            "@app/domainV2/*": ["./src/app/domainV2/*"],
+            "@app/glossaryV2/*": ["./src/app/glossaryV2/*"],
+            "@app/homeV2/*": ["./src/app/homeV2/*"],
+            "@app/lineageV2/*": ["./src/app/lineageV2/*"],
+            "@app/previewV2/*": ["./src/app/previewV2/*"],
+            "@app/sharedV2/*": ["./src/app/sharedV2/*"],
+            "@types": ["./src/types.generated.ts"],
+            "@images": ["./src/images"]
+        }
     },
-    "include": ["src", "src/conf/theme/styled-components.d.ts", "vite.config.ts", ".eslintrc.js"]
+    "include": ["src", "src/conf/theme/styled-components.d.ts", "vite.config.ts", ".eslintrc.js", "functions"]
 }
diff --git a/datahub-web-react/vite.config.ts b/datahub-web-react/vite.config.ts
index 683b37974c85a..2532b24067754 100644
--- a/datahub-web-react/vite.config.ts
+++ b/datahub-web-react/vite.config.ts
@@ -97,5 +97,33 @@ export default defineConfig(({ mode }) => {
                 exclude: [],
             },
         },
+        resolve: {
+            alias: {
+                // Root Directories
+                '@src': path.resolve(__dirname, '/src'),
+                '@app': path.resolve(__dirname, '/src/app'),
+                '@conf': path.resolve(__dirname, '/src/conf'),
+                '@components': path.resolve(__dirname, 'src/alchemy-components'),
+                '@graphql': path.resolve(__dirname, 'src/graphql'),
+                '@graphql-mock': path.resolve(__dirname, 'src/graphql-mock'),
+                '@images': path.resolve(__dirname, 'src/images'),
+                '@providers': path.resolve(__dirname, 'src/providers'),
+                '@utils': path.resolve(__dirname, 'src/utils'),
+
+                // App Specific Directories
+                '@app/entityV1': path.resolve(__dirname, 'src/app/entity'),
+                '@app/entityV2': path.resolve(__dirname, 'src/app/entityV2'),
+                '@app/searchV2': path.resolve(__dirname, 'src/app/searchV2'),
+                '@app/domainV2': path.resolve(__dirname, 'src/app/domainV2'),
+                '@app/glossaryV2': path.resolve(__dirname, 'src/app/glossaryV2'),
+                '@app/homeV2': path.resolve(__dirname, 'src/app/homeV2'),
+                '@app/lineageV2': path.resolve(__dirname, 'src/app/lineageV2'),
+                '@app/previewV2': path.resolve(__dirname, 'src/app/previewV2'),
+                '@app/sharedV2': path.resolve(__dirname, 'src/app/sharedV2'),
+
+                // Specific Files
+                '@types': path.resolve(__dirname, 'src/types.generated.ts'),
+            },
+        },
     };
 });
diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock
index 9dc563c958dd1..dc7260efd183f 100644
--- a/datahub-web-react/yarn.lock
+++ b/datahub-web-react/yarn.lock
@@ -7,6 +7,11 @@
   resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.2.tgz#a6abc715fb6884851fca9dad37fc34739a04fd11"
   integrity sha512-DA5a1C0gD/pLOvhv33YMrbf2FK3oUzwNl9oOJqE4XVjuEtt6XIakRcsd7eLiOSPkp1kTRQGICTA8cKra/vFbjw==
 
+"@adobe/css-tools@^4.4.0":
+  version "4.4.1"
+  resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.4.1.tgz#2447a230bfe072c1659e6815129c03cf170710e3"
+  integrity sha512-12WGKBQzjUAI4ayyF4IAtfw2QR/IDoqk6jTddXDhtYTJF9ASmoE1zst7cVtP0aL/F1jUJL5r+JxKXKEgHNbEUQ==
+
 "@ampproject/remapping@^2.2.0":
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630"
@@ -195,6 +200,15 @@
     "@babel/highlight" "^7.23.4"
     chalk "^2.4.2"
 
+"@babel/code-frame@^7.25.9", "@babel/code-frame@^7.26.0":
+  version "7.26.2"
+  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.26.2.tgz#4b5fab97d33338eff916235055f0ebc21e573a85"
+  integrity sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==
+  dependencies:
+    "@babel/helper-validator-identifier" "^7.25.9"
+    js-tokens "^4.0.0"
+    picocolors "^1.0.0"
+
 "@babel/compat-data@^7.20.5", "@babel/compat-data@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.5.tgz#b1f6c86a02d85d2dd3368a2b67c09add8cd0c255"
@@ -205,6 +219,11 @@
   resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.5.tgz#ffb878728bb6bdcb6f4510aa51b1be9afb8cfd98"
   integrity sha512-uU27kfDRlhfKl+w1U6vp16IuvSLtjAxdArVXPa9BvLkrr7CYIsxH5adpHObeAGY/41+syctUWOZ140a2Rvkgjw==
 
+"@babel/compat-data@^7.25.9":
+  version "7.26.3"
+  resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.26.3.tgz#99488264a56b2aded63983abd6a417f03b92ed02"
+  integrity sha512-nHIxvKPniQXpmQLb0vhY3VaFb3S0YrTAwpOWJZh1wn3oJPjJk9Asva204PsBdmAE8vpzfHudT8DB0scYvy9q0g==
+
 "@babel/core@^7.14.0":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.5.tgz#d67d9747ecf26ee7ecd3ebae1ee22225fe902a89"
@@ -247,6 +266,27 @@
     json5 "^2.2.3"
     semver "^6.3.1"
 
+"@babel/core@^7.18.9":
+  version "7.26.0"
+  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.26.0.tgz#d78b6023cc8f3114ccf049eb219613f74a747b40"
+  integrity sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==
+  dependencies:
+    "@ampproject/remapping" "^2.2.0"
+    "@babel/code-frame" "^7.26.0"
+    "@babel/generator" "^7.26.0"
+    "@babel/helper-compilation-targets" "^7.25.9"
+    "@babel/helper-module-transforms" "^7.26.0"
+    "@babel/helpers" "^7.26.0"
+    "@babel/parser" "^7.26.0"
+    "@babel/template" "^7.25.9"
+    "@babel/traverse" "^7.25.9"
+    "@babel/types" "^7.26.0"
+    convert-source-map "^2.0.0"
+    debug "^4.1.0"
+    gensync "^1.0.0-beta.2"
+    json5 "^2.2.3"
+    semver "^6.3.1"
+
 "@babel/core@^7.22.9":
   version "7.23.5"
   resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.5.tgz#6e23f2acbcb77ad283c5ed141f824fd9f70101c7"
@@ -298,6 +338,17 @@
     "@jridgewell/trace-mapping" "^0.3.17"
     jsesc "^2.5.1"
 
+"@babel/generator@^7.26.0":
+  version "7.26.3"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.26.3.tgz#ab8d4360544a425c90c248df7059881f4b2ce019"
+  integrity sha512-6FF/urZvD0sTeO7k6/B15pMLC4CHUv1426lzr3N01aHJTl046uCAh9LXW/fzeXXjPNCJ6iABW5XaWOsIZB93aQ==
+  dependencies:
+    "@babel/parser" "^7.26.3"
+    "@babel/types" "^7.26.3"
+    "@jridgewell/gen-mapping" "^0.3.5"
+    "@jridgewell/trace-mapping" "^0.3.25"
+    jsesc "^3.0.2"
+
 "@babel/helper-annotate-as-pure@^7.0.0", "@babel/helper-annotate-as-pure@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz#e7f06737b197d580a01edf75d97e2c8be99d3882"
@@ -327,6 +378,17 @@
     lru-cache "^5.1.1"
     semver "^6.3.1"
 
+"@babel/helper-compilation-targets@^7.25.9":
+  version "7.25.9"
+  resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz#55af025ce365be3cdc0c1c1e56c6af617ce88875"
+  integrity sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==
+  dependencies:
+    "@babel/compat-data" "^7.25.9"
+    "@babel/helper-validator-option" "^7.25.9"
+    browserslist "^4.24.0"
+    lru-cache "^5.1.1"
+    semver "^6.3.1"
+
 "@babel/helper-create-class-features-plugin@^7.18.6":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.22.5.tgz#2192a1970ece4685fbff85b48da2c32fcb130b7c"
@@ -396,6 +458,14 @@
   dependencies:
     "@babel/types" "^7.22.15"
 
+"@babel/helper-module-imports@^7.25.9":
+  version "7.25.9"
+  resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz#e7f8d20602ebdbf9ebbea0a0751fb0f2a4141715"
+  integrity sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==
+  dependencies:
+    "@babel/traverse" "^7.25.9"
+    "@babel/types" "^7.25.9"
+
 "@babel/helper-module-transforms@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.5.tgz#0f65daa0716961b6e96b164034e737f60a80d2ef"
@@ -432,6 +502,15 @@
     "@babel/helper-split-export-declaration" "^7.22.6"
     "@babel/helper-validator-identifier" "^7.22.20"
 
+"@babel/helper-module-transforms@^7.26.0":
+  version "7.26.0"
+  resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz#8ce54ec9d592695e58d84cd884b7b5c6a2fdeeae"
+  integrity sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==
+  dependencies:
+    "@babel/helper-module-imports" "^7.25.9"
+    "@babel/helper-validator-identifier" "^7.25.9"
+    "@babel/traverse" "^7.25.9"
+
 "@babel/helper-optimise-call-expression@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.22.5.tgz#f21531a9ccbff644fdd156b4077c16ff0c3f609e"
@@ -494,6 +573,11 @@
   resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz#9478c707febcbbe1ddb38a3d91a2e054ae622d83"
   integrity sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==
 
+"@babel/helper-string-parser@^7.25.9":
+  version "7.25.9"
+  resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz#1aabb72ee72ed35789b4bbcad3ca2862ce614e8c"
+  integrity sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==
+
 "@babel/helper-validator-identifier@^7.22.20":
   version "7.22.20"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0"
@@ -504,6 +588,11 @@
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193"
   integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==
 
+"@babel/helper-validator-identifier@^7.25.9":
+  version "7.25.9"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz#24b64e2c3ec7cd3b3c547729b8d16871f22cbdc7"
+  integrity sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==
+
 "@babel/helper-validator-option@^7.22.15":
   version "7.23.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.23.5.tgz#907a3fbd4523426285365d1206c423c4c5520307"
@@ -514,6 +603,11 @@
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.5.tgz#de52000a15a177413c8234fa3a8af4ee8102d0ac"
   integrity sha512-R3oB6xlIVKUnxNUxbmgq7pKjxpru24zlimpE8WK47fACIlM0II/Hm1RS8IaOI7NgCr6LNS+jl5l75m20npAziw==
 
+"@babel/helper-validator-option@^7.25.9":
+  version "7.25.9"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz#86e45bd8a49ab7e03f276577f96179653d41da72"
+  integrity sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==
+
 "@babel/helpers@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.22.5.tgz#74bb4373eb390d1ceed74a15ef97767e63120820"
@@ -541,6 +635,14 @@
     "@babel/traverse" "^7.23.5"
     "@babel/types" "^7.23.5"
 
+"@babel/helpers@^7.26.0":
+  version "7.26.0"
+  resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.26.0.tgz#30e621f1eba5aa45fe6f4868d2e9154d884119a4"
+  integrity sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==
+  dependencies:
+    "@babel/template" "^7.25.9"
+    "@babel/types" "^7.26.0"
+
 "@babel/highlight@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.5.tgz#aa6c05c5407a67ebce408162b7ede789b4d22031"
@@ -574,6 +676,13 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719"
   integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==
 
+"@babel/parser@^7.25.9", "@babel/parser@^7.26.0", "@babel/parser@^7.26.3":
+  version "7.26.3"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.26.3.tgz#8c51c5db6ddf08134af1ddbacf16aaab48bac234"
+  integrity sha512-WJ/CvmY8Mea8iDXo6a7RK2wbmJITT5fN3BEkRuFlxVyNx8jOKIIhmC4fSkTcPcf8JyavbBwIe6OpiCOBXt/IcA==
+  dependencies:
+    "@babel/types" "^7.26.3"
+
 "@babel/plugin-proposal-class-properties@^7.0.0":
   version "7.18.6"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz#b110f59741895f7ec21a6fff696ec46265c446a3"
@@ -823,6 +932,13 @@
   dependencies:
     regenerator-runtime "^0.13.11"
 
+"@babel/runtime@^7.17.8", "@babel/runtime@^7.23.9":
+  version "7.26.0"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.0.tgz#8600c2f595f277c60815256418b85356a65173c1"
+  integrity sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==
+  dependencies:
+    regenerator-runtime "^0.14.0"
+
 "@babel/template@^7.18.10", "@babel/template@^7.22.15":
   version "7.22.15"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38"
@@ -841,7 +957,16 @@
     "@babel/parser" "^7.22.5"
     "@babel/types" "^7.22.5"
 
-"@babel/traverse@>=7.23.2", "@babel/traverse@^7.1.6", "@babel/traverse@^7.14.0", "@babel/traverse@^7.16.8", "@babel/traverse@^7.22.5", "@babel/traverse@^7.23.2", "@babel/traverse@^7.23.5", "@babel/traverse@^7.4.5":
+"@babel/template@^7.25.9":
+  version "7.25.9"
+  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.9.tgz#ecb62d81a8a6f5dc5fe8abfc3901fc52ddf15016"
+  integrity sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==
+  dependencies:
+    "@babel/code-frame" "^7.25.9"
+    "@babel/parser" "^7.25.9"
+    "@babel/types" "^7.25.9"
+
+"@babel/traverse@>=7.23.2", "@babel/traverse@^7.1.6", "@babel/traverse@^7.14.0", "@babel/traverse@^7.16.8", "@babel/traverse@^7.18.9", "@babel/traverse@^7.22.5", "@babel/traverse@^7.23.2", "@babel/traverse@^7.23.5", "@babel/traverse@^7.25.9", "@babel/traverse@^7.4.5":
   version "7.23.5"
   resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.5.tgz#f546bf9aba9ef2b042c0e00d245990c15508e7ec"
   integrity sha512-czx7Xy5a6sapWWRx61m1Ke1Ra4vczu1mCTtJam5zRTBOonfdJ+S/B6HYmGYu3fJtr8GGET3si6IhgWVBhJ/m8w==
@@ -875,6 +1000,14 @@
     "@babel/helper-validator-identifier" "^7.22.20"
     to-fast-properties "^2.0.0"
 
+"@babel/types@^7.18.9", "@babel/types@^7.25.9", "@babel/types@^7.26.0", "@babel/types@^7.26.3":
+  version "7.26.3"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.26.3.tgz#37e79830f04c2b5687acc77db97fbc75fb81f3c0"
+  integrity sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==
+  dependencies:
+    "@babel/helper-string-parser" "^7.25.9"
+    "@babel/helper-validator-identifier" "^7.25.9"
+
 "@babel/types@^7.21.3":
   version "7.23.6"
   resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.6.tgz#be33fdb151e1f5a56877d704492c240fc71c7ccd"
@@ -918,6 +1051,17 @@
     "@emotion/weak-memoize" "^0.3.0"
     stylis "4.1.3"
 
+"@emotion/cache@^11.11.0":
+  version "11.13.5"
+  resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.13.5.tgz#e78dad0489e1ed7572507ba8ed9d2130529e4266"
+  integrity sha512-Z3xbtJ+UcK76eWkagZ1onvn/wAVb1GOMuR15s30Fm2wrMgC7jzpnO2JZXr4eujTTqoQFUrZIw/rT0c6Zzjca1g==
+  dependencies:
+    "@emotion/memoize" "^0.9.0"
+    "@emotion/sheet" "^1.4.0"
+    "@emotion/utils" "^1.4.2"
+    "@emotion/weak-memoize" "^0.4.0"
+    stylis "4.2.0"
+
 "@emotion/css@^11.1.3":
   version "11.10.5"
   resolved "https://registry.yarnpkg.com/@emotion/css/-/css-11.10.5.tgz#ca01bb83ce60517bc3a5c01d27ccf552fed84d9d"
@@ -958,6 +1102,11 @@
   resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.8.0.tgz#f580f9beb67176fa57aae70b08ed510e1b18980f"
   integrity sha512-G/YwXTkv7Den9mXDO7AhLWkE3q+I92B+VqAE+dYG4NGPaHZGvt3G8Q0p9vmE+sq7rTGphUbAvmQ9YpbfMQGGlA==
 
+"@emotion/memoize@^0.9.0":
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.9.0.tgz#745969d649977776b43fc7648c556aaa462b4102"
+  integrity sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==
+
 "@emotion/react@^11.9.0":
   version "11.10.5"
   resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.10.5.tgz#95fff612a5de1efa9c0d535384d3cfa115fe175d"
@@ -988,6 +1137,11 @@
   resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.2.1.tgz#0767e0305230e894897cadb6c8df2c51e61a6c2c"
   integrity sha512-zxRBwl93sHMsOj4zs+OslQKg/uhF38MB+OMKoCrVuS0nyTkqnau+BM3WGEoOptg9Oz45T/aIGs1qbVAsEFo3nA==
 
+"@emotion/sheet@^1.4.0":
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.4.0.tgz#c9299c34d248bc26e82563735f78953d2efca83c"
+  integrity sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==
+
 "@emotion/styled@^11.3.0":
   version "11.10.5"
   resolved "https://registry.yarnpkg.com/@emotion/styled/-/styled-11.10.5.tgz#1fe7bf941b0909802cb826457e362444e7e96a79"
@@ -1025,121 +1179,251 @@
   resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.2.0.tgz#9716eaccbc6b5ded2ea5a90d65562609aab0f561"
   integrity sha512-sn3WH53Kzpw8oQ5mgMmIzzyAaH2ZqFEbozVVBSYp538E06OSE6ytOp7pRAjNQR+Q/orwqdQYJSe2m3hCOeznkw==
 
+"@emotion/utils@^1.4.2":
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.4.2.tgz#6df6c45881fcb1c412d6688a311a98b7f59c1b52"
+  integrity sha512-3vLclRofFziIa3J2wDh9jjbkUz9qk5Vi3IZ/FSTKViB0k+ef0fPV7dYrUIugbgupYDx7v9ud/SjrtEP8Y4xLoA==
+
 "@emotion/weak-memoize@^0.3.0":
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.3.0.tgz#ea89004119dc42db2e1dba0f97d553f7372f6fcb"
   integrity sha512-AHPmaAx+RYfZz0eYu6Gviiagpmiyw98ySSlQvCUhVGDRtDFe4DBS0x1bSjdF3gqUDYOczB+yYvBTtEylYSdRhg==
 
+"@emotion/weak-memoize@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz#5e13fac887f08c44f76b0ccaf3370eb00fec9bb6"
+  integrity sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==
+
+"@esbuild/aix-ppc64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.24.0.tgz#b57697945b50e99007b4c2521507dc613d4a648c"
+  integrity sha512-WtKdFM7ls47zkKHFVzMz8opM7LkcsIp9amDUBIAWirg70RM71WRSjdILPsY5Uv1D42ZpUfaPILDlfactHgsRkw==
+
 "@esbuild/android-arm64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz#984b4f9c8d0377443cc2dfcef266d02244593622"
   integrity sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==
 
+"@esbuild/android-arm64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.24.0.tgz#1add7e0af67acefd556e407f8497e81fddad79c0"
+  integrity sha512-Vsm497xFM7tTIPYK9bNTYJyF/lsP590Qc1WxJdlB6ljCbdZKU9SY8i7+Iin4kyhV/KV5J2rOKsBQbB77Ab7L/w==
+
 "@esbuild/android-arm@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.18.20.tgz#fedb265bc3a589c84cc11f810804f234947c3682"
   integrity sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==
 
+"@esbuild/android-arm@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.24.0.tgz#ab7263045fa8e090833a8e3c393b60d59a789810"
+  integrity sha512-arAtTPo76fJ/ICkXWetLCc9EwEHKaeya4vMrReVlEIUCAUncH7M4bhMQ+M9Vf+FFOZJdTNMXNBrWwW+OXWpSew==
+
 "@esbuild/android-x64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.18.20.tgz#35cf419c4cfc8babe8893d296cd990e9e9f756f2"
   integrity sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==
 
+"@esbuild/android-x64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.24.0.tgz#e8f8b196cfdfdd5aeaebbdb0110983460440e705"
+  integrity sha512-t8GrvnFkiIY7pa7mMgJd7p8p8qqYIz1NYiAoKc75Zyv73L3DZW++oYMSHPRarcotTKuSs6m3hTOa5CKHaS02TQ==
+
 "@esbuild/darwin-arm64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz#08172cbeccf95fbc383399a7f39cfbddaeb0d7c1"
   integrity sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==
 
+"@esbuild/darwin-arm64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.24.0.tgz#2d0d9414f2acbffd2d86e98253914fca603a53dd"
+  integrity sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw==
+
 "@esbuild/darwin-x64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz#d70d5790d8bf475556b67d0f8b7c5bdff053d85d"
   integrity sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==
 
+"@esbuild/darwin-x64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.24.0.tgz#33087aab31a1eb64c89daf3d2cf8ce1775656107"
+  integrity sha512-rgtz6flkVkh58od4PwTRqxbKH9cOjaXCMZgWD905JOzjFKW+7EiUObfd/Kav+A6Gyud6WZk9w+xu6QLytdi2OA==
+
 "@esbuild/freebsd-arm64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz#98755cd12707f93f210e2494d6a4b51b96977f54"
   integrity sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==
 
+"@esbuild/freebsd-arm64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.0.tgz#bb76e5ea9e97fa3c753472f19421075d3a33e8a7"
+  integrity sha512-6Mtdq5nHggwfDNLAHkPlyLBpE5L6hwsuXZX8XNmHno9JuL2+bg2BX5tRkwjyfn6sKbxZTq68suOjgWqCicvPXA==
+
 "@esbuild/freebsd-x64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz#c1eb2bff03915f87c29cece4c1a7fa1f423b066e"
   integrity sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==
 
+"@esbuild/freebsd-x64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.24.0.tgz#e0e2ce9249fdf6ee29e5dc3d420c7007fa579b93"
+  integrity sha512-D3H+xh3/zphoX8ck4S2RxKR6gHlHDXXzOf6f/9dbFt/NRBDIE33+cVa49Kil4WUjxMGW0ZIYBYtaGCa2+OsQwQ==
+
 "@esbuild/linux-arm64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz#bad4238bd8f4fc25b5a021280c770ab5fc3a02a0"
   integrity sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==
 
+"@esbuild/linux-arm64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.24.0.tgz#d1b2aa58085f73ecf45533c07c82d81235388e75"
+  integrity sha512-TDijPXTOeE3eaMkRYpcy3LarIg13dS9wWHRdwYRnzlwlA370rNdZqbcp0WTyyV/k2zSxfko52+C7jU5F9Tfj1g==
+
 "@esbuild/linux-arm@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz#3e617c61f33508a27150ee417543c8ab5acc73b0"
   integrity sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==
 
+"@esbuild/linux-arm@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.24.0.tgz#8e4915df8ea3e12b690a057e77a47b1d5935ef6d"
+  integrity sha512-gJKIi2IjRo5G6Glxb8d3DzYXlxdEj2NlkixPsqePSZMhLudqPhtZ4BUrpIuTjJYXxvF9njql+vRjB2oaC9XpBw==
+
 "@esbuild/linux-ia32@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz#699391cccba9aee6019b7f9892eb99219f1570a7"
   integrity sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==
 
+"@esbuild/linux-ia32@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.24.0.tgz#8200b1110666c39ab316572324b7af63d82013fb"
+  integrity sha512-K40ip1LAcA0byL05TbCQ4yJ4swvnbzHscRmUilrmP9Am7//0UjPreh4lpYzvThT2Quw66MhjG//20mrufm40mA==
+
 "@esbuild/linux-loong64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz#e6fccb7aac178dd2ffb9860465ac89d7f23b977d"
   integrity sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==
 
+"@esbuild/linux-loong64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.24.0.tgz#6ff0c99cf647504df321d0640f0d32e557da745c"
+  integrity sha512-0mswrYP/9ai+CU0BzBfPMZ8RVm3RGAN/lmOMgW4aFUSOQBjA31UP8Mr6DDhWSuMwj7jaWOT0p0WoZ6jeHhrD7g==
+
 "@esbuild/linux-mips64el@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz#eeff3a937de9c2310de30622a957ad1bd9183231"
   integrity sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==
 
+"@esbuild/linux-mips64el@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.24.0.tgz#3f720ccd4d59bfeb4c2ce276a46b77ad380fa1f3"
+  integrity sha512-hIKvXm0/3w/5+RDtCJeXqMZGkI2s4oMUGj3/jM0QzhgIASWrGO5/RlzAzm5nNh/awHE0A19h/CvHQe6FaBNrRA==
+
 "@esbuild/linux-ppc64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz#2f7156bde20b01527993e6881435ad79ba9599fb"
   integrity sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==
 
+"@esbuild/linux-ppc64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.24.0.tgz#9d6b188b15c25afd2e213474bf5f31e42e3aa09e"
+  integrity sha512-HcZh5BNq0aC52UoocJxaKORfFODWXZxtBaaZNuN3PUX3MoDsChsZqopzi5UupRhPHSEHotoiptqikjN/B77mYQ==
+
 "@esbuild/linux-riscv64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz#6628389f210123d8b4743045af8caa7d4ddfc7a6"
   integrity sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==
 
+"@esbuild/linux-riscv64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.24.0.tgz#f989fdc9752dfda286c9cd87c46248e4dfecbc25"
+  integrity sha512-bEh7dMn/h3QxeR2KTy1DUszQjUrIHPZKyO6aN1X4BCnhfYhuQqedHaa5MxSQA/06j3GpiIlFGSsy1c7Gf9padw==
+
 "@esbuild/linux-s390x@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz#255e81fb289b101026131858ab99fba63dcf0071"
   integrity sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==
 
+"@esbuild/linux-s390x@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.24.0.tgz#29ebf87e4132ea659c1489fce63cd8509d1c7319"
+  integrity sha512-ZcQ6+qRkw1UcZGPyrCiHHkmBaj9SiCD8Oqd556HldP+QlpUIe2Wgn3ehQGVoPOvZvtHm8HPx+bH20c9pvbkX3g==
+
 "@esbuild/linux-x64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz#c7690b3417af318a9b6f96df3031a8865176d338"
   integrity sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==
 
+"@esbuild/linux-x64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.24.0.tgz#4af48c5c0479569b1f359ffbce22d15f261c0cef"
+  integrity sha512-vbutsFqQ+foy3wSSbmjBXXIJ6PL3scghJoM8zCL142cGaZKAdCZHyf+Bpu/MmX9zT9Q0zFBVKb36Ma5Fzfa8xA==
+
 "@esbuild/netbsd-x64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz#30e8cd8a3dded63975e2df2438ca109601ebe0d1"
   integrity sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==
 
+"@esbuild/netbsd-x64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.24.0.tgz#1ae73d23cc044a0ebd4f198334416fb26c31366c"
+  integrity sha512-hjQ0R/ulkO8fCYFsG0FZoH+pWgTTDreqpqY7UnQntnaKv95uP5iW3+dChxnx7C3trQQU40S+OgWhUVwCjVFLvg==
+
+"@esbuild/openbsd-arm64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.0.tgz#5d904a4f5158c89859fd902c427f96d6a9e632e2"
+  integrity sha512-MD9uzzkPQbYehwcN583yx3Tu5M8EIoTD+tUgKF982WYL9Pf5rKy9ltgD0eUgs8pvKnmizxjXZyLt0z6DC3rRXg==
+
 "@esbuild/openbsd-x64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz#7812af31b205055874c8082ea9cf9ab0da6217ae"
   integrity sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==
 
+"@esbuild/openbsd-x64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.24.0.tgz#4c8aa88c49187c601bae2971e71c6dc5e0ad1cdf"
+  integrity sha512-4ir0aY1NGUhIC1hdoCzr1+5b43mw99uNwVzhIq1OY3QcEwPDO3B7WNXBzaKY5Nsf1+N11i1eOfFcq+D/gOS15Q==
+
 "@esbuild/sunos-x64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz#d5c275c3b4e73c9b0ecd38d1ca62c020f887ab9d"
   integrity sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==
 
+"@esbuild/sunos-x64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.24.0.tgz#8ddc35a0ea38575fa44eda30a5ee01ae2fa54dd4"
+  integrity sha512-jVzdzsbM5xrotH+W5f1s+JtUy1UWgjU0Cf4wMvffTB8m6wP5/kx0KiaLHlbJO+dMgtxKV8RQ/JvtlFcdZ1zCPA==
+
 "@esbuild/win32-arm64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz#73bc7f5a9f8a77805f357fab97f290d0e4820ac9"
   integrity sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==
 
+"@esbuild/win32-arm64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.24.0.tgz#6e79c8543f282c4539db684a207ae0e174a9007b"
+  integrity sha512-iKc8GAslzRpBytO2/aN3d2yb2z8XTVfNV0PjGlCxKo5SgWmNXx82I/Q3aG1tFfS+A2igVCY97TJ8tnYwpUWLCA==
+
 "@esbuild/win32-ia32@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz#ec93cbf0ef1085cc12e71e0d661d20569ff42102"
   integrity sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==
 
+"@esbuild/win32-ia32@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.24.0.tgz#057af345da256b7192d18b676a02e95d0fa39103"
+  integrity sha512-vQW36KZolfIudCcTnaTpmLQ24Ha1RjygBo39/aLkM2kmjkWmZGEJ5Gn9l5/7tzXA42QGIoWbICfg6KLLkIw6yw==
+
 "@esbuild/win32-x64@0.18.20":
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz#786c5f41f043b07afb1af37683d7c33668858f6d"
   integrity sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==
 
+"@esbuild/win32-x64@0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.24.0.tgz#168ab1c7e1c318b922637fad8f339d48b01e1244"
+  integrity sha512-7IAFPrjSQIJrGsK6flwg7NFmwBoSTyF3rl7If0hNUFQU4ilTsEPL6GuMuU9BfIWVVGuRnuIidkSMC+c0Otu8IA==
+
 "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0":
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59"
@@ -1172,6 +1456,16 @@
   resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.43.0.tgz#559ca3d9ddbd6bf907ad524320a0d14b85586af0"
   integrity sha512-s2UHCoiXfxMvmfzqoN+vrQ84ahUSYde9qNO1MdxmoEhyHWsfmwOpFlwYV+ePJEVc7gFnATGUi376WowX1N7tFg==
 
+"@fontsource/mulish@^5.0.16":
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/@fontsource/mulish/-/mulish-5.1.0.tgz#e5cfb25d3dc83e0f4f0c828708b304bf33c9f7a4"
+  integrity sha512-dETFYrY9aGE1aD8g3xSkpyAir2YDjrnYsujZrZj1C/3gv3iKlr2MUQYUFRTCdcnoFLMXpq2izDW/644OGgYsPQ==
+
+"@geometricpanda/storybook-addon-badges@^2.0.2":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@geometricpanda/storybook-addon-badges/-/storybook-addon-badges-2.0.5.tgz#eea1c27df8956eddeca612a82e5606034232da95"
+  integrity sha512-FH56ly6ZhltjyKQWxUKORP67BxhL9FMJRByS5lqKZpeP8J2MMsMXG7eQmFXKcZGQORfVQye+1uYYWXweDOiFTQ==
+
 "@graphql-codegen/add@^2.0.2":
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/@graphql-codegen/add/-/add-2.0.2.tgz#4acbb95be9ebb859a3cebfe7132fdf49ffe06dd8"
@@ -1654,6 +1948,14 @@
   dependencies:
     "@sinclair/typebox" "^0.27.8"
 
+"@joshwooding/vite-plugin-react-docgen-typescript@0.4.2":
+  version "0.4.2"
+  resolved "https://registry.yarnpkg.com/@joshwooding/vite-plugin-react-docgen-typescript/-/vite-plugin-react-docgen-typescript-0.4.2.tgz#c2591d2d7b02160341672d6bf3cc248dd60f2530"
+  integrity sha512-feQ+ntr+8hbVudnsTUapiMN9q8T90XA1d5jn9QzY09sNoj4iD9wi0PY1vsBFTda4ZjEaxRK9S81oarR2nj7TFQ==
+  dependencies:
+    magic-string "^0.27.0"
+    react-docgen-typescript "^2.2.2"
+
 "@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2":
   version "0.3.3"
   resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098"
@@ -1663,21 +1965,45 @@
     "@jridgewell/sourcemap-codec" "^1.4.10"
     "@jridgewell/trace-mapping" "^0.3.9"
 
+"@jridgewell/gen-mapping@^0.3.5":
+  version "0.3.5"
+  resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36"
+  integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==
+  dependencies:
+    "@jridgewell/set-array" "^1.2.1"
+    "@jridgewell/sourcemap-codec" "^1.4.10"
+    "@jridgewell/trace-mapping" "^0.3.24"
+
 "@jridgewell/resolve-uri@3.1.0":
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78"
   integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==
 
+"@jridgewell/resolve-uri@^3.1.0":
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6"
+  integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==
+
 "@jridgewell/set-array@^1.0.1":
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72"
   integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==
 
+"@jridgewell/set-array@^1.2.1":
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280"
+  integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==
+
 "@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10":
   version "1.4.14"
   resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
   integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
 
+"@jridgewell/sourcemap-codec@^1.4.13", "@jridgewell/sourcemap-codec@^1.4.14", "@jridgewell/sourcemap-codec@^1.5.0":
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz#3188bcb273a414b0d215fd22a58540b989b9409a"
+  integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==
+
 "@jridgewell/sourcemap-codec@^1.4.15":
   version "1.4.15"
   resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32"
@@ -1691,6 +2017,14 @@
     "@jridgewell/resolve-uri" "3.1.0"
     "@jridgewell/sourcemap-codec" "1.4.14"
 
+"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25":
+  version "0.3.25"
+  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0"
+  integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==
+  dependencies:
+    "@jridgewell/resolve-uri" "^3.1.0"
+    "@jridgewell/sourcemap-codec" "^1.4.14"
+
 "@linaria/core@3.0.0-beta.13":
   version "3.0.0-beta.13"
   resolved "https://registry.yarnpkg.com/@linaria/core/-/core-3.0.0-beta.13.tgz#049c5be5faa67e341e413a0f6b641d5d78d91056"
@@ -1719,6 +2053,13 @@
     refractor "^3.3.1"
     unist-util-visit "^2.0.3"
 
+"@mdx-js/react@^3.0.0":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@mdx-js/react/-/react-3.1.0.tgz#c4522e335b3897b9a845db1dbdd2f966ae8fb0ed"
+  integrity sha512-QjHtSaoameoalGnKDT3FoIl4+9RwyTmo9ZJGBdLOks/YOiWHoRDI3PUwEzOE7kEmGcV3AFcp9K6dYu9rEuKLAQ==
+  dependencies:
+    "@types/mdx" "^2.0.0"
+
 "@monaco-editor/loader@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@monaco-editor/loader/-/loader-1.2.0.tgz#373fad69973384624e3d9b60eefd786461a76acd"
@@ -1753,6 +2094,36 @@
   resolved "https://registry.yarnpkg.com/@mui/core-downloads-tracker/-/core-downloads-tracker-5.10.15.tgz#490f3dea5327c892f063496a0219c48301da0fa0"
   integrity sha512-xFcS0LpdF0Q1qJrrNsYUv9PU+ovvhCEPTOMw2jcpEFtl3CA87dLpvztORR5oE2UBFjWF7qLQLOwboQU1+xC7Cw==
 
+"@mui/core-downloads-tracker@^5.16.9":
+  version "5.16.9"
+  resolved "https://registry.yarnpkg.com/@mui/core-downloads-tracker/-/core-downloads-tracker-5.16.9.tgz#df68e6b685857d48cab5ed73ec2bf47fdd48d252"
+  integrity sha512-ue3j79XJ56+F6DlTtFTM+n//5AvNENOvl3MFruZZP5iZzz+hOq6WBwnr+YxiMlr+kvmMHuHxgOHFdPR8+mElDw==
+
+"@mui/icons-material@^5.15.21":
+  version "5.16.9"
+  resolved "https://registry.yarnpkg.com/@mui/icons-material/-/icons-material-5.16.9.tgz#0f754ef0b943e3bea78343cd2861e8d95a114617"
+  integrity sha512-nnOJIqan6FS6zEsLX3vf8LZ4vXpZjP5xfCFezeXmqfQConypCOZG4nangoVwKwROlas7b6/bqOdacFUb/HuM/g==
+  dependencies:
+    "@babel/runtime" "^7.23.9"
+
+"@mui/material@^5.15.21":
+  version "5.16.9"
+  resolved "https://registry.yarnpkg.com/@mui/material/-/material-5.16.9.tgz#d0d3e2e9882d11d94171298e2d7ab38db736ce1f"
+  integrity sha512-XC0oHFm7mrWV0tvhed9uv/o6kLNClnLj1eo/ufuKbj+rgk47ek8Y6HjHe3cGvMn4Bcq8KyoQPgzdwqvS2ZzYrA==
+  dependencies:
+    "@babel/runtime" "^7.23.9"
+    "@mui/core-downloads-tracker" "^5.16.9"
+    "@mui/system" "^5.16.8"
+    "@mui/types" "^7.2.15"
+    "@mui/utils" "^5.16.8"
+    "@popperjs/core" "^2.11.8"
+    "@types/react-transition-group" "^4.4.10"
+    clsx "^2.1.0"
+    csstype "^3.1.3"
+    prop-types "^15.8.1"
+    react-is "^18.3.1"
+    react-transition-group "^4.4.5"
+
 "@mui/material@^5.8.5":
   version "5.10.15"
   resolved "https://registry.yarnpkg.com/@mui/material/-/material-5.10.15.tgz#37345f5a3d71c662703af7b5be0cca229b2a1416"
@@ -1780,6 +2151,15 @@
     "@mui/utils" "^5.10.15"
     prop-types "^15.8.1"
 
+"@mui/private-theming@^5.16.8":
+  version "5.16.8"
+  resolved "https://registry.yarnpkg.com/@mui/private-theming/-/private-theming-5.16.8.tgz#7914996caaf6eedc59914aeab83dcd2d4e4da1ec"
+  integrity sha512-3Vl9yFVLU6T3CFtxRMQTcJ60Ijv7wxQi4yjH92+9YXcsqvVspeIYoocqNoIV/1bXGYfyWu5zrCmwQVHaGY7bug==
+  dependencies:
+    "@babel/runtime" "^7.23.9"
+    "@mui/utils" "^5.16.8"
+    prop-types "^15.8.1"
+
 "@mui/styled-engine@^5.10.14":
   version "5.10.14"
   resolved "https://registry.yarnpkg.com/@mui/styled-engine/-/styled-engine-5.10.14.tgz#4395198a1919254a3edabf6e8fc8d43c9c59b5c3"
@@ -1790,6 +2170,16 @@
     csstype "^3.1.1"
     prop-types "^15.8.1"
 
+"@mui/styled-engine@^5.16.8":
+  version "5.16.8"
+  resolved "https://registry.yarnpkg.com/@mui/styled-engine/-/styled-engine-5.16.8.tgz#b8ca35f93f503a51d0759a05475bfd28e10757ea"
+  integrity sha512-OFdgFf8JczSRs0kvWGdSn0ZeXxWrY0LITDPJ/nAtLEvUUTyrlFaO4il3SECX8ruzvf1VnAxHx4M/4mX9oOn9yA==
+  dependencies:
+    "@babel/runtime" "^7.23.9"
+    "@emotion/cache" "^11.11.0"
+    csstype "^3.1.3"
+    prop-types "^15.8.1"
+
 "@mui/system@^5.10.15":
   version "5.10.15"
   resolved "https://registry.yarnpkg.com/@mui/system/-/system-5.10.15.tgz#4bb58d1d1a531137559b775038a18d6050d9ee57"
@@ -1804,11 +2194,30 @@
     csstype "^3.1.1"
     prop-types "^15.8.1"
 
+"@mui/system@^5.16.8":
+  version "5.16.8"
+  resolved "https://registry.yarnpkg.com/@mui/system/-/system-5.16.8.tgz#e5010d76cd2fdcc403ad3f98abfba99d330055ad"
+  integrity sha512-L32TaFDFpGIi1g6ysRtmhc9zDgrlxDXu3NlrGE8gAsQw/ziHrPdr0PNr20O0POUshA1q14W4dNZ/z0Nx2F9lhA==
+  dependencies:
+    "@babel/runtime" "^7.23.9"
+    "@mui/private-theming" "^5.16.8"
+    "@mui/styled-engine" "^5.16.8"
+    "@mui/types" "^7.2.15"
+    "@mui/utils" "^5.16.8"
+    clsx "^2.1.0"
+    csstype "^3.1.3"
+    prop-types "^15.8.1"
+
 "@mui/types@^7.2.1":
   version "7.2.1"
   resolved "https://registry.yarnpkg.com/@mui/types/-/types-7.2.1.tgz#1eb2bc182c595029884047f2525ad4dbefea318e"
   integrity sha512-c5mSM7ivD8EsqK6HUi9hQPr5V7TJ/IRThUQ9nWNYPdhCGriTSQV4vL6DflT99LkM+wLiIS1rVjphpEWxERep7A==
 
+"@mui/types@^7.2.15":
+  version "7.2.19"
+  resolved "https://registry.yarnpkg.com/@mui/types/-/types-7.2.19.tgz#c941954dd24393fdce5f07830d44440cf4ab6c80"
+  integrity sha512-6XpZEM/Q3epK9RN8ENoXuygnqUQxE+siN/6rGRi2iwJPgBUR25mphYQ9ZI87plGh58YoZ5pp40bFvKYOCDJ3tA==
+
 "@mui/utils@^5.10.15":
   version "5.10.15"
   resolved "https://registry.yarnpkg.com/@mui/utils/-/utils-5.10.15.tgz#54fc1b373508d20dd5568070b2dcc0818e6bebba"
@@ -1820,6 +2229,18 @@
     prop-types "^15.8.1"
     react-is "^18.2.0"
 
+"@mui/utils@^5.16.8":
+  version "5.16.8"
+  resolved "https://registry.yarnpkg.com/@mui/utils/-/utils-5.16.8.tgz#e44acf38d446d361347c46b3e81ae366f615f37b"
+  integrity sha512-P/yb7BSWallQUeiNGxb+TM8epHteIUC8gzNTdPV2VfKhVY/EnGliHgt5np0GPkjQ7EzwDi/+gBevrAJtf+K94A==
+  dependencies:
+    "@babel/runtime" "^7.23.9"
+    "@mui/types" "^7.2.15"
+    "@types/prop-types" "^15.7.12"
+    clsx "^2.1.1"
+    prop-types "^15.8.1"
+    react-is "^18.3.1"
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -1880,6 +2301,11 @@
   resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.6.tgz#cee20bd55e68a1720bdab363ecf0c821ded4cd45"
   integrity sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==
 
+"@popperjs/core@^2.11.8":
+  version "2.11.8"
+  resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.8.tgz#6b79032e760a0899cd4204710beede972a3a185f"
+  integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==
+
 "@rc-component/portal@^1.0.0-6", "@rc-component/portal@^1.0.0-8", "@rc-component/portal@^1.0.2":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@rc-component/portal/-/portal-1.1.0.tgz#6b94450d2c2b00d50b141bd7a0be23bd96503dbe"
@@ -2866,6 +3292,15 @@
   resolved "https://registry.yarnpkg.com/@repeaterjs/repeater/-/repeater-3.0.5.tgz#b77571685410217a548a9c753aa3cdfc215bfc78"
   integrity sha512-l3YHBLAol6d/IKnB9LhpD0cEZWAoe3eFKUyTYWmFmCO2Q/WOckxLQAUyMZWwZV2M/m3+4vgRoaolFqaII82/TA==
 
+"@rollup/pluginutils@^5.0.2":
+  version "5.1.3"
+  resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-5.1.3.tgz#3001bf1a03f3ad24457591f2c259c8e514e0dbdf"
+  integrity sha512-Pnsb6f32CD2W3uCaLZIzDmeFyQ2b8UWMFI7xtwUezpcGBDVDW6y9XgAWIlARiGAo6eNF5FK5aQTr0LFyNyqq5A==
+  dependencies:
+    "@types/estree" "^1.0.0"
+    estree-walker "^2.0.2"
+    picomatch "^4.0.2"
+
 "@rollup/pluginutils@^5.0.4":
   version "5.0.5"
   resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-5.0.5.tgz#bbb4c175e19ebfeeb8c132c2eea0ecb89941a66c"
@@ -2890,6 +3325,259 @@
   resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e"
   integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==
 
+"@storybook/addon-actions@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-actions/-/addon-actions-8.4.7.tgz#210c6bb5a7e17c3664c300b4b69b6243ec34b9cd"
+  integrity sha512-mjtD5JxcPuW74T6h7nqMxWTvDneFtokg88p6kQ5OnC1M259iAXb//yiSZgu/quunMHPCXSiqn4FNOSgASTSbsA==
+  dependencies:
+    "@storybook/global" "^5.0.0"
+    "@types/uuid" "^9.0.1"
+    dequal "^2.0.2"
+    polished "^4.2.2"
+    uuid "^9.0.0"
+
+"@storybook/addon-backgrounds@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-backgrounds/-/addon-backgrounds-8.4.7.tgz#56856bdafc5a2ba18cc19422320883c9e8f66c1c"
+  integrity sha512-I4/aErqtFiazcoWyKafOAm3bLpxTj6eQuH/woSbk1Yx+EzN+Dbrgx1Updy8//bsNtKkcrXETITreqHC+a57DHQ==
+  dependencies:
+    "@storybook/global" "^5.0.0"
+    memoizerific "^1.11.3"
+    ts-dedent "^2.0.0"
+
+"@storybook/addon-controls@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-controls/-/addon-controls-8.4.7.tgz#0c2ace0c7056248577f08f90471f29e861b485be"
+  integrity sha512-377uo5IsJgXLnQLJixa47+11V+7Wn9KcDEw+96aGCBCfLbWNH8S08tJHHnSu+jXg9zoqCAC23MetntVp6LetHA==
+  dependencies:
+    "@storybook/global" "^5.0.0"
+    dequal "^2.0.2"
+    ts-dedent "^2.0.0"
+
+"@storybook/addon-docs@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-docs/-/addon-docs-8.4.7.tgz#556515da1049f97023427301e11ecb52d0b9dbe7"
+  integrity sha512-NwWaiTDT5puCBSUOVuf6ME7Zsbwz7Y79WF5tMZBx/sLQ60vpmJVQsap6NSjvK1Ravhc21EsIXqemAcBjAWu80w==
+  dependencies:
+    "@mdx-js/react" "^3.0.0"
+    "@storybook/blocks" "8.4.7"
+    "@storybook/csf-plugin" "8.4.7"
+    "@storybook/react-dom-shim" "8.4.7"
+    react "^16.8.0 || ^17.0.0 || ^18.0.0"
+    react-dom "^16.8.0 || ^17.0.0 || ^18.0.0"
+    ts-dedent "^2.0.0"
+
+"@storybook/addon-essentials@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-essentials/-/addon-essentials-8.4.7.tgz#381c74230d1b1a209d5fdc017d241c016b98affe"
+  integrity sha512-+BtZHCBrYtQKILtejKxh0CDRGIgTl9PumfBOKRaihYb4FX1IjSAxoV/oo/IfEjlkF5f87vouShWsRa8EUauFDw==
+  dependencies:
+    "@storybook/addon-actions" "8.4.7"
+    "@storybook/addon-backgrounds" "8.4.7"
+    "@storybook/addon-controls" "8.4.7"
+    "@storybook/addon-docs" "8.4.7"
+    "@storybook/addon-highlight" "8.4.7"
+    "@storybook/addon-measure" "8.4.7"
+    "@storybook/addon-outline" "8.4.7"
+    "@storybook/addon-toolbars" "8.4.7"
+    "@storybook/addon-viewport" "8.4.7"
+    ts-dedent "^2.0.0"
+
+"@storybook/addon-highlight@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-highlight/-/addon-highlight-8.4.7.tgz#06b9752977e38884007e9446f9a2b0c04c873229"
+  integrity sha512-whQIDBd3PfVwcUCrRXvCUHWClXe9mQ7XkTPCdPo4B/tZ6Z9c6zD8JUHT76ddyHivixFLowMnA8PxMU6kCMAiNw==
+  dependencies:
+    "@storybook/global" "^5.0.0"
+
+"@storybook/addon-interactions@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-interactions/-/addon-interactions-8.4.7.tgz#d34545db5ea6f03a5499ad6742c3317fb9e02d55"
+  integrity sha512-fnufT3ym8ht3HHUIRVXAH47iOJW/QOb0VSM+j269gDuvyDcY03D1civCu1v+eZLGaXPKJ8vtjr0L8zKQ/4P0JQ==
+  dependencies:
+    "@storybook/global" "^5.0.0"
+    "@storybook/instrumenter" "8.4.7"
+    "@storybook/test" "8.4.7"
+    polished "^4.2.2"
+    ts-dedent "^2.2.0"
+
+"@storybook/addon-links@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-links/-/addon-links-8.4.7.tgz#c38b2b63c3b0308adacff4b0758464a0657154c4"
+  integrity sha512-L/1h4dMeMKF+MM0DanN24v5p3faNYbbtOApMgg7SlcBT/tgo3+cAjkgmNpYA8XtKnDezm+T2mTDhB8mmIRZpIQ==
+  dependencies:
+    "@storybook/csf" "^0.1.11"
+    "@storybook/global" "^5.0.0"
+    ts-dedent "^2.0.0"
+
+"@storybook/addon-measure@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-measure/-/addon-measure-8.4.7.tgz#9d556ba34b57c13ad8d00bd953b27ec405a64d23"
+  integrity sha512-QfvqYWDSI5F68mKvafEmZic3SMiK7zZM8VA0kTXx55hF/+vx61Mm0HccApUT96xCXIgmwQwDvn9gS4TkX81Dmw==
+  dependencies:
+    "@storybook/global" "^5.0.0"
+    tiny-invariant "^1.3.1"
+
+"@storybook/addon-onboarding@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-onboarding/-/addon-onboarding-8.4.7.tgz#212a5e27db1ee8440a2dd0d5c67ac29f0e6efda5"
+  integrity sha512-FdC2NV60VNYeMxf6DVe0qV9ucSBAzMh1//C0Qqwq8CcjthMbmKlVZ7DqbVsbIHKnFaSCaUC88eR5olAfMaauCQ==
+  dependencies:
+    react-confetti "^6.1.0"
+
+"@storybook/addon-outline@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-outline/-/addon-outline-8.4.7.tgz#8a35fe519dd639bb287a370da2222e6ffdce4020"
+  integrity sha512-6LYRqUZxSodmAIl8icr585Oi8pmzbZ90aloZJIpve+dBAzo7ydYrSQxxoQEVltXbKf3VeVcrs64ouAYqjisMYA==
+  dependencies:
+    "@storybook/global" "^5.0.0"
+    ts-dedent "^2.0.0"
+
+"@storybook/addon-toolbars@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-toolbars/-/addon-toolbars-8.4.7.tgz#b898d4deaf6f5a58f3b70bd8d136cd4ec2844b79"
+  integrity sha512-OSfdv5UZs+NdGB+nZmbafGUWimiweJ/56gShlw8Neo/4jOJl1R3rnRqqY7MYx8E4GwoX+i3GF5C3iWFNQqlDcw==
+
+"@storybook/addon-viewport@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-viewport/-/addon-viewport-8.4.7.tgz#e65c53608f52149c06347b395487960605fc4805"
+  integrity sha512-hvczh/jjuXXcOogih09a663sRDDSATXwbE866al1DXgbDFraYD/LxX/QDb38W9hdjU9+Qhx8VFIcNWoMQns5HQ==
+  dependencies:
+    memoizerific "^1.11.3"
+
+"@storybook/blocks@8.4.7", "@storybook/blocks@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/blocks/-/blocks-8.4.7.tgz#ee17f59dd52d11c97c39b0f6b03957085a80ad95"
+  integrity sha512-+QH7+JwXXXIyP3fRCxz/7E2VZepAanXJM7G8nbR3wWsqWgrRp4Wra6MvybxAYCxU7aNfJX5c+RW84SNikFpcIA==
+  dependencies:
+    "@storybook/csf" "^0.1.11"
+    "@storybook/icons" "^1.2.12"
+    ts-dedent "^2.0.0"
+
+"@storybook/builder-vite@8.4.7", "@storybook/builder-vite@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/builder-vite/-/builder-vite-8.4.7.tgz#3d6d542fa1f46fce5ee7a159dc8491cb4421254d"
+  integrity sha512-LovyXG5VM0w7CovI/k56ZZyWCveQFVDl0m7WwetpmMh2mmFJ+uPQ35BBsgTvTfc8RHi+9Q3F58qP1MQSByXi9g==
+  dependencies:
+    "@storybook/csf-plugin" "8.4.7"
+    browser-assert "^1.2.1"
+    ts-dedent "^2.0.0"
+
+"@storybook/components@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/components/-/components-8.4.7.tgz#09eeffa07aa672ad3966ca1764a43003731b1d30"
+  integrity sha512-uyJIcoyeMWKAvjrG9tJBUCKxr2WZk+PomgrgrUwejkIfXMO76i6jw9BwLa0NZjYdlthDv30r9FfbYZyeNPmF0g==
+
+"@storybook/core@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/core/-/core-8.4.7.tgz#af9cbb3f26f0b6c98c679a134ce776c202570d66"
+  integrity sha512-7Z8Z0A+1YnhrrSXoKKwFFI4gnsLbWzr8fnDCU6+6HlDukFYh8GHRcZ9zKfqmy6U3hw2h8H5DrHsxWfyaYUUOoA==
+  dependencies:
+    "@storybook/csf" "^0.1.11"
+    better-opn "^3.0.2"
+    browser-assert "^1.2.1"
+    esbuild "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0"
+    esbuild-register "^3.5.0"
+    jsdoc-type-pratt-parser "^4.0.0"
+    process "^0.11.10"
+    recast "^0.23.5"
+    semver "^7.6.2"
+    util "^0.12.5"
+    ws "^8.2.3"
+
+"@storybook/csf-plugin@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/csf-plugin/-/csf-plugin-8.4.7.tgz#0117c872b05bf033eec089ab0224e0fab01da810"
+  integrity sha512-Fgogplu4HImgC+AYDcdGm1rmL6OR1rVdNX1Be9C/NEXwOCpbbBwi0BxTf/2ZxHRk9fCeaPEcOdP5S8QHfltc1g==
+  dependencies:
+    unplugin "^1.3.1"
+
+"@storybook/csf@^0.1.11":
+  version "0.1.12"
+  resolved "https://registry.yarnpkg.com/@storybook/csf/-/csf-0.1.12.tgz#1dcfa0f398a69b834c563884b5f747db3d5a81df"
+  integrity sha512-9/exVhabisyIVL0VxTCxo01Tdm8wefIXKXfltAPTSr8cbLn5JAxGQ6QV3mjdecLGEOucfoVhAKtJfVHxEK1iqw==
+  dependencies:
+    type-fest "^2.19.0"
+
+"@storybook/global@^5.0.0":
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/@storybook/global/-/global-5.0.0.tgz#b793d34b94f572c1d7d9e0f44fac4e0dbc9572ed"
+  integrity sha512-FcOqPAXACP0I3oJ/ws6/rrPT9WGhu915Cg8D02a9YxLo0DE9zI+a9A5gRGvmQ09fiWPukqI8ZAEoQEdWUKMQdQ==
+
+"@storybook/icons@^1.2.12":
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/@storybook/icons/-/icons-1.3.0.tgz#a5c1460fb15a7260e0b638ab86163f7347a0061e"
+  integrity sha512-Nz/UzeYQdUZUhacrPyfkiiysSjydyjgg/p0P9HxB4p/WaJUUjMAcaoaLgy3EXx61zZJ3iD36WPuDkZs5QYrA0A==
+
+"@storybook/instrumenter@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/instrumenter/-/instrumenter-8.4.7.tgz#5a37876fee8f828241a1e7fd76891c6effc1805a"
+  integrity sha512-k6NSD3jaRCCHAFtqXZ7tw8jAzD/yTEWXGya+REgZqq5RCkmJ+9S4Ytp/6OhQMPtPFX23gAuJJzTQVLcCr+gjRg==
+  dependencies:
+    "@storybook/global" "^5.0.0"
+    "@vitest/utils" "^2.1.1"
+
+"@storybook/manager-api@8.4.7", "@storybook/manager-api@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/manager-api/-/manager-api-8.4.7.tgz#4e13debf645c9300d7d6d49195e720d0c7ecd261"
+  integrity sha512-ELqemTviCxAsZ5tqUz39sDmQkvhVAvAgiplYy9Uf15kO0SP2+HKsCMzlrm2ue2FfkUNyqbDayCPPCB0Cdn/mpQ==
+
+"@storybook/preview-api@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/preview-api/-/preview-api-8.4.7.tgz#85e01a97f4182b974581765d725f6c7a7d190013"
+  integrity sha512-0QVQwHw+OyZGHAJEXo6Knx+6/4er7n2rTDE5RYJ9F2E2Lg42E19pfdLlq2Jhoods2Xrclo3wj6GWR//Ahi39Eg==
+
+"@storybook/react-dom-shim@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/react-dom-shim/-/react-dom-shim-8.4.7.tgz#f0dd5bbf2fc185def72d9d08a11c8de22f152c2a"
+  integrity sha512-6bkG2jvKTmWrmVzCgwpTxwIugd7Lu+2btsLAqhQSzDyIj2/uhMNp8xIMr/NBDtLgq3nomt9gefNa9xxLwk/OMg==
+
+"@storybook/react-vite@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/react-vite/-/react-vite-8.4.7.tgz#1a755596d65551c77850361da76df47027687664"
+  integrity sha512-iiY9iLdMXhDnilCEVxU6vQsN72pW3miaf0WSenOZRyZv3HdbpgOxI0qapOS0KCyRUnX9vTlmrSPTMchY4cAeOg==
+  dependencies:
+    "@joshwooding/vite-plugin-react-docgen-typescript" "0.4.2"
+    "@rollup/pluginutils" "^5.0.2"
+    "@storybook/builder-vite" "8.4.7"
+    "@storybook/react" "8.4.7"
+    find-up "^5.0.0"
+    magic-string "^0.30.0"
+    react-docgen "^7.0.0"
+    resolve "^1.22.8"
+    tsconfig-paths "^4.2.0"
+
+"@storybook/react@8.4.7":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/react/-/react-8.4.7.tgz#e2cf62b3c1d8e4bfe5eff82ced07ec473d4e4fd1"
+  integrity sha512-nQ0/7i2DkaCb7dy0NaT95llRVNYWQiPIVuhNfjr1mVhEP7XD090p0g7eqUmsx8vfdHh2BzWEo6CoBFRd3+EXxw==
+  dependencies:
+    "@storybook/components" "8.4.7"
+    "@storybook/global" "^5.0.0"
+    "@storybook/manager-api" "8.4.7"
+    "@storybook/preview-api" "8.4.7"
+    "@storybook/react-dom-shim" "8.4.7"
+    "@storybook/theming" "8.4.7"
+
+"@storybook/test@8.4.7", "@storybook/test@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/test/-/test-8.4.7.tgz#7f58f2cdf3a6d810bf3ff4e0e2fee634040c678f"
+  integrity sha512-AhvJsu5zl3uG40itSQVuSy5WByp3UVhS6xAnme4FWRwgSxhvZjATJ3AZkkHWOYjnnk+P2/sbz/XuPli1FVCWoQ==
+  dependencies:
+    "@storybook/csf" "^0.1.11"
+    "@storybook/global" "^5.0.0"
+    "@storybook/instrumenter" "8.4.7"
+    "@testing-library/dom" "10.4.0"
+    "@testing-library/jest-dom" "6.5.0"
+    "@testing-library/user-event" "14.5.2"
+    "@vitest/expect" "2.0.5"
+    "@vitest/spy" "2.0.5"
+
+"@storybook/theming@8.4.7", "@storybook/theming@^8.1.11":
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/@storybook/theming/-/theming-8.4.7.tgz#c308f6a883999bd35e87826738ab8a76515932b5"
+  integrity sha512-99rgLEjf7iwfSEmdqlHkSG3AyLcK0sfExcr0jnc6rLiAkBhzuIsvcHjjUwkR210SOCgXqBPW0ZA6uhnuyppHLw==
+
 "@svgmoji/blob@^3.2.0":
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/@svgmoji/blob/-/blob-3.2.0.tgz#62a0ab1ba22a0d27f23cb38aacf6d4fb13123dfb"
@@ -3017,6 +3705,20 @@
     "@svgr/hast-util-to-babel-ast" "8.0.0"
     svg-parser "^2.0.4"
 
+"@testing-library/dom@10.4.0":
+  version "10.4.0"
+  resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-10.4.0.tgz#82a9d9462f11d240ecadbf406607c6ceeeff43a8"
+  integrity sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==
+  dependencies:
+    "@babel/code-frame" "^7.10.4"
+    "@babel/runtime" "^7.12.5"
+    "@types/aria-query" "^5.0.1"
+    aria-query "5.3.0"
+    chalk "^4.1.0"
+    dom-accessibility-api "^0.5.9"
+    lz-string "^1.5.0"
+    pretty-format "^27.0.2"
+
 "@testing-library/dom@^8.0.0":
   version "8.20.1"
   resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-8.20.1.tgz#2e52a32e46fc88369eef7eef634ac2a192decd9f"
@@ -3031,6 +3733,19 @@
     lz-string "^1.5.0"
     pretty-format "^27.0.2"
 
+"@testing-library/jest-dom@6.5.0":
+  version "6.5.0"
+  resolved "https://registry.yarnpkg.com/@testing-library/jest-dom/-/jest-dom-6.5.0.tgz#50484da3f80fb222a853479f618a9ce5c47bfe54"
+  integrity sha512-xGGHpBXYSHUUr6XsKBfs85TWlYKpTc37cSBBVrXcib2MkHLboWlkClhWF37JKlDb9KEq3dHs+f2xR7XJEWGBxA==
+  dependencies:
+    "@adobe/css-tools" "^4.4.0"
+    aria-query "^5.0.0"
+    chalk "^3.0.0"
+    css.escape "^1.5.1"
+    dom-accessibility-api "^0.6.3"
+    lodash "^4.17.21"
+    redent "^3.0.0"
+
 "@testing-library/jest-dom@^6.1.4":
   version "6.1.4"
   resolved "https://registry.yarnpkg.com/@testing-library/jest-dom/-/jest-dom-6.1.4.tgz#cf0835c33bc5ef00befb9e672b1e3e6a710e30e3"
@@ -3054,6 +3769,11 @@
     "@testing-library/dom" "^8.0.0"
     "@types/react-dom" "<18.0.0"
 
+"@testing-library/user-event@14.5.2":
+  version "14.5.2"
+  resolved "https://registry.yarnpkg.com/@testing-library/user-event/-/user-event-14.5.2.tgz#db7257d727c891905947bd1c1a99da20e03c2ebd"
+  integrity sha512-YAh82Wh4TIrxYLmfGcixwD18oIjyC1pFQC2Y01F2lzV2HTMiYrI0nze0FD0ocB//CKS/7jIUgae+adPqxK5yCQ==
+
 "@tommoor/remove-markdown@^0.3.2":
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/@tommoor/remove-markdown/-/remove-markdown-0.3.2.tgz#5288ddd0e26b6b173e76ebb31c94653b0dcff45d"
@@ -3080,6 +3800,17 @@
     "@types/babel__template" "*"
     "@types/babel__traverse" "*"
 
+"@types/babel__core@^7.18.0":
+  version "7.20.5"
+  resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.5.tgz#3df15f27ba85319caa07ba08d0721889bb39c017"
+  integrity sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==
+  dependencies:
+    "@babel/parser" "^7.20.7"
+    "@babel/types" "^7.20.7"
+    "@types/babel__generator" "*"
+    "@types/babel__template" "*"
+    "@types/babel__traverse" "*"
+
 "@types/babel__generator@*":
   version "7.6.2"
   resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.2.tgz#f3d71178e187858f7c45e30380f8f1b7415a12d8"
@@ -3102,6 +3833,13 @@
   dependencies:
     "@babel/types" "^7.3.0"
 
+"@types/babel__traverse@^7.18.0":
+  version "7.20.6"
+  resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.6.tgz#8dc9f0ae0f202c08d8d4dab648912c8d6038e3f7"
+  integrity sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==
+  dependencies:
+    "@babel/types" "^7.20.7"
+
 "@types/chai-subset@^1.3.3":
   version "1.3.4"
   resolved "https://registry.yarnpkg.com/@types/chai-subset/-/chai-subset-1.3.4.tgz#7938fa929dd12db451457e4d6faa27bcd599a729"
@@ -3192,6 +3930,11 @@
   resolved "https://registry.yarnpkg.com/@types/direction/-/direction-1.0.0.tgz#6a0962feade8502f9e986e87abe1130b611b13be"
   integrity sha512-et1wmqXm/5smJ8lTJfBnwD12/2Y7eVJLKbuaRT0h2xaKAoo1h8Dz2Io22GObDLFwxY1ddXRTLH3Gq5v44Fl/2w==
 
+"@types/doctrine@^0.0.9":
+  version "0.0.9"
+  resolved "https://registry.yarnpkg.com/@types/doctrine/-/doctrine-0.0.9.tgz#d86a5f452a15e3e3113b99e39616a9baa0f9863f"
+  integrity sha512-eOIHzCUSH7SMfonMG1LsC2f8vxBFtho6NGBznK41R84YzPuvSBzrhEps33IsQiOW9+VL6NQ9DbjQJznk/S4uRA==
+
 "@types/dompurify@^2.3.3":
   version "2.3.3"
   resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-2.3.3.tgz#c24c92f698f77ed9cc9d9fa7888f90cf2bfaa23f"
@@ -3278,6 +4021,11 @@
   dependencies:
     "@types/unist" "*"
 
+"@types/mdx@^2.0.0":
+  version "2.0.13"
+  resolved "https://registry.yarnpkg.com/@types/mdx/-/mdx-2.0.13.tgz#68f6877043d377092890ff5b298152b0a21671bd"
+  integrity sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw==
+
 "@types/min-document@^2.19.0":
   version "2.19.0"
   resolved "https://registry.yarnpkg.com/@types/min-document/-/min-document-2.19.0.tgz#4f9919e789917c00de967a2c38fa8d234cbcd7d6"
@@ -3325,6 +4073,11 @@
   resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.5.tgz#5f19d2b85a98e9558036f6a3cacc8819420f05cf"
   integrity sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==
 
+"@types/prop-types@^15.7.12":
+  version "15.7.14"
+  resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.14.tgz#1433419d73b2a7ebfc6918dcefd2ec0d5cd698f2"
+  integrity sha512-gNMvNH49DJ7OJYv+KAKn0Xp45p8PLl6zo2YnvDIbTd4J6MER2BmWN49TG7n9LvkyihINxeKW8+3bfS2yDC9dzQ==
+
 "@types/query-string@^6.3.0":
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/@types/query-string/-/query-string-6.3.0.tgz#b6fa172a01405abcaedac681118e78429d62ea39"
@@ -3390,6 +4143,13 @@
     "@types/history" "*"
     "@types/react" "*"
 
+"@types/react-transition-group@^4.4.10":
+  version "4.4.11"
+  resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.11.tgz#d963253a611d757de01ebb241143b1017d5d63d5"
+  integrity sha512-RM05tAniPZ5DZPzzNFP+DmrcOdD0efDUxMy3145oljWSl3x9ZV5vhme98gTxFrj2lhXvmGNnUiuDyJgY9IKkNA==
+  dependencies:
+    "@types/react" "*"
+
 "@types/react-transition-group@^4.4.5":
   version "4.4.5"
   resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.5.tgz#aae20dcf773c5aa275d5b9f7cdbca638abc5e416"
@@ -3429,6 +4189,11 @@
   dependencies:
     "@types/prismjs" "*"
 
+"@types/resolve@^1.20.2":
+  version "1.20.6"
+  resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.20.6.tgz#e6e60dad29c2c8c206c026e6dd8d6d1bdda850b8"
+  integrity sha512-A4STmOXPhMUtHH+S6ymgE2GiBSMqf4oTvcQZMcHzokuTLVYzXTB8ttjcgxOVaAp2lGwEdzZ0J+cRbbeevQj1UQ==
+
 "@types/scheduler@*":
   version "0.16.1"
   resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.1.tgz#18845205e86ff0038517aab7a18a62a6b9f71275"
@@ -3485,6 +4250,11 @@
   resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e"
   integrity sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==
 
+"@types/uuid@^9.0.1":
+  version "9.0.8"
+  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba"
+  integrity sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==
+
 "@types/ws@^8.0.0":
   version "8.5.10"
   resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.10.tgz#4acfb517970853fa6574a3a6886791d04a396787"
@@ -3734,6 +4504,14 @@
     d3-shape "^1.2.0"
     prop-types "^15.6.2"
 
+"@visx/gradient@^3.3.0":
+  version "3.12.0"
+  resolved "https://registry.yarnpkg.com/@visx/gradient/-/gradient-3.12.0.tgz#4898241cdd78ff3003dae584787a4b2dffff1eb7"
+  integrity sha512-QRatjjdUEPbcp4pqRca1JlChpAnmmIAO3r3ZscLK7D1xEIANlIjzjl3uNgrmseYmBAYyPCcJH8Zru07R97ovOg==
+  dependencies:
+    "@types/react" "*"
+    prop-types "^15.5.7"
+
 "@visx/grid@3.2.0":
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/@visx/grid/-/grid-3.2.0.tgz#1db936e0c20bf45d2e8280f8c70b62982c67fb59"
@@ -3945,6 +4723,30 @@
     "@vitest/utils" "0.34.6"
     chai "^4.3.10"
 
+"@vitest/expect@2.0.5":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-2.0.5.tgz#f3745a6a2c18acbea4d39f5935e913f40d26fa86"
+  integrity sha512-yHZtwuP7JZivj65Gxoi8upUN2OzHTi3zVfjwdpu2WrvCZPLwsJ2Ey5ILIPccoW23dd/zQBlJ4/dhi7DWNyXCpA==
+  dependencies:
+    "@vitest/spy" "2.0.5"
+    "@vitest/utils" "2.0.5"
+    chai "^5.1.1"
+    tinyrainbow "^1.2.0"
+
+"@vitest/pretty-format@2.0.5":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@vitest/pretty-format/-/pretty-format-2.0.5.tgz#91d2e6d3a7235c742e1a6cc50e7786e2f2979b1e"
+  integrity sha512-h8k+1oWHfwTkyTkb9egzwNMfJAEx4veaPSnMeKbVSjp4euqGSbQlm5+6VHwTr7u4FJslVVsUG5nopCaAYdOmSQ==
+  dependencies:
+    tinyrainbow "^1.2.0"
+
+"@vitest/pretty-format@2.1.8":
+  version "2.1.8"
+  resolved "https://registry.yarnpkg.com/@vitest/pretty-format/-/pretty-format-2.1.8.tgz#88f47726e5d0cf4ba873d50c135b02e4395e2bca"
+  integrity sha512-9HiSZ9zpqNLKlbIDRWOnAWqgcA7xu+8YxXSekhr0Ykab7PAYFkhkwoqVArPOtJhPmYeE2YHgKZlj3CP36z2AJQ==
+  dependencies:
+    tinyrainbow "^1.2.0"
+
 "@vitest/runner@0.34.6":
   version "0.34.6"
   resolved "https://registry.yarnpkg.com/@vitest/runner/-/runner-0.34.6.tgz#6f43ca241fc96b2edf230db58bcde5b974b8dcaf"
@@ -3970,6 +4772,13 @@
   dependencies:
     tinyspy "^2.1.1"
 
+"@vitest/spy@2.0.5":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@vitest/spy/-/spy-2.0.5.tgz#590fc07df84a78b8e9dd976ec2090920084a2b9f"
+  integrity sha512-c/jdthAhvJdpfVuaexSrnawxZz6pywlTPe84LUB2m/4t3rl2fTo9NFGBG4oWgaD+FTgDDV8hJ/nibT7IfH3JfA==
+  dependencies:
+    tinyspy "^3.0.0"
+
 "@vitest/utils@0.34.6":
   version "0.34.6"
   resolved "https://registry.yarnpkg.com/@vitest/utils/-/utils-0.34.6.tgz#38a0a7eedddb8e7291af09a2409cb8a189516968"
@@ -3979,6 +4788,25 @@
     loupe "^2.3.6"
     pretty-format "^29.5.0"
 
+"@vitest/utils@2.0.5":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@vitest/utils/-/utils-2.0.5.tgz#6f8307a4b6bc6ceb9270007f73c67c915944e926"
+  integrity sha512-d8HKbqIcya+GR67mkZbrzhS5kKhtp8dQLcmRZLGTscGVg7yImT82cIrhtn2L8+VujWcy6KZweApgNmPsTAO/UQ==
+  dependencies:
+    "@vitest/pretty-format" "2.0.5"
+    estree-walker "^3.0.3"
+    loupe "^3.1.1"
+    tinyrainbow "^1.2.0"
+
+"@vitest/utils@^2.1.1":
+  version "2.1.8"
+  resolved "https://registry.yarnpkg.com/@vitest/utils/-/utils-2.1.8.tgz#f8ef85525f3362ebd37fd25d268745108d6ae388"
+  integrity sha512-dwSoui6djdwbfFmIgbIjX2ZhIoG7Ex/+xpxyiEgIGzjliY8xGkcpITKTlp6B4MgtGkF2ilvm97cPM96XZaAgcA==
+  dependencies:
+    "@vitest/pretty-format" "2.1.8"
+    loupe "^3.1.2"
+    tinyrainbow "^1.2.0"
+
 "@whatwg-node/events@^0.0.3":
   version "0.0.3"
   resolved "https://registry.yarnpkg.com/@whatwg-node/events/-/events-0.0.3.tgz#13a65dd4f5893f55280f766e29ae48074927acad"
@@ -4085,6 +4913,11 @@ acorn@^8.10.0, acorn@^8.9.0:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.2.tgz#ca0d78b51895be5390a5903c5b3bdcdaf78ae40b"
   integrity sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==
 
+acorn@^8.14.0:
+  version "8.14.0"
+  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.14.0.tgz#063e2c70cac5fb4f6467f0b11152e04c682795b0"
+  integrity sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==
+
 acorn@^8.8.0:
   version "8.9.0"
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.9.0.tgz#78a16e3b2bcc198c10822786fa6679e245db5b59"
@@ -4245,7 +5078,7 @@ aria-query@5.1.3:
   dependencies:
     deep-equal "^2.0.5"
 
-aria-query@^5.0.0, aria-query@^5.1.3:
+aria-query@5.3.0, aria-query@^5.0.0, aria-query@^5.1.3:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.3.0.tgz#650c569e41ad90b51b3d7df5e5eed1c7549c103e"
   integrity sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==
@@ -4331,11 +5164,23 @@ assertion-error@^1.1.0:
   resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b"
   integrity sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==
 
+assertion-error@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-2.0.1.tgz#f641a196b335690b1070bf00b6e7593fec190bf7"
+  integrity sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==
+
 ast-types-flow@^0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/ast-types-flow/-/ast-types-flow-0.0.7.tgz#f70b735c6bca1a5c9c22d982c3e39e7feba3bdad"
   integrity sha1-9wtzXGvKGlycItmCw+Oef+ujva0=
 
+ast-types@^0.16.1:
+  version "0.16.1"
+  resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.16.1.tgz#7a9da1617c9081bc121faafe91711b4c8bb81da2"
+  integrity sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==
+  dependencies:
+    tslib "^2.0.1"
+
 astral-regex@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31"
@@ -4366,6 +5211,13 @@ available-typed-arrays@^1.0.5:
   resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7"
   integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==
 
+available-typed-arrays@^1.0.7:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz#a5cc375d6a03c2efc87a553f3e0b1522def14846"
+  integrity sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==
+  dependencies:
+    possible-typed-array-names "^1.0.0"
+
 axe-core@^4.6.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.7.2.tgz#040a7342b20765cb18bb50b628394c21bccc17a0"
@@ -4478,6 +5330,13 @@ base64-js@^1.3.1:
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
 
+better-opn@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/better-opn/-/better-opn-3.0.2.tgz#f96f35deaaf8f34144a4102651babcf00d1d8817"
+  integrity sha512-aVNobHnJqLiUelTaHat9DZ1qM2w0C0Eym4LPI/3JxOnSokGVdsl1T1kN7TFvsEAD8G47A6VKQ0TVHqbBnYMJlQ==
+  dependencies:
+    open "^8.0.4"
+
 binary-extensions@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
@@ -4524,6 +5383,11 @@ braces@~3.0.2:
   dependencies:
     fill-range "^7.0.1"
 
+browser-assert@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/browser-assert/-/browser-assert-1.2.1.tgz#9aaa5a2a8c74685c2ae05bfe46efd606f068c200"
+  integrity sha512-nfulgvOR6S4gt9UKCeGJOuSGBPGiFT6oQ/2UBnvTY/5aQ1PnksW72fhZkM30DzoRRv2WpwZf1vHHEr3mtuXIWQ==
+
 browserslist@^4.21.3:
   version "4.21.9"
   resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.9.tgz#e11bdd3c313d7e2a9e87e8b4b0c7872b13897635"
@@ -4544,6 +5408,16 @@ browserslist@^4.21.9:
     node-releases "^2.0.13"
     update-browserslist-db "^1.0.13"
 
+browserslist@^4.24.0:
+  version "4.24.2"
+  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.24.2.tgz#f5845bc91069dbd55ee89faf9822e1d885d16580"
+  integrity sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==
+  dependencies:
+    caniuse-lite "^1.0.30001669"
+    electron-to-chromium "^1.5.41"
+    node-releases "^2.0.18"
+    update-browserslist-db "^1.1.1"
+
 bser@2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05"
@@ -4576,6 +5450,14 @@ cac@^6.7.14:
   resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959"
   integrity sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==
 
+call-bind-apply-helpers@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.0.tgz#33127b42608972f76812a501d69db5d8ce404979"
+  integrity sha512-CCKAP2tkPau7D3GE8+V8R6sQubA9R5foIzGp+85EXCVSCivuxBNAWqcpn72PKYiIcqoViv/kcUDpaEIMBVi1lQ==
+  dependencies:
+    es-errors "^1.3.0"
+    function-bind "^1.1.2"
+
 call-bind@^1.0.0, call-bind@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c"
@@ -4593,6 +5475,16 @@ call-bind@^1.0.4:
     get-intrinsic "^1.2.1"
     set-function-length "^1.1.1"
 
+call-bind@^1.0.7:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.8.tgz#0736a9660f537e3388826f440d5ec45f744eaa4c"
+  integrity sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==
+  dependencies:
+    call-bind-apply-helpers "^1.0.0"
+    es-define-property "^1.0.0"
+    get-intrinsic "^1.2.4"
+    set-function-length "^1.2.2"
+
 callsites@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
@@ -4631,6 +5523,11 @@ caniuse-lite@^1.0.30001541:
   resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001559.tgz#95a982440d3d314c471db68d02664fb7536c5a30"
   integrity sha512-cPiMKZgqgkg5LY3/ntGeLFUpi6tzddBNS58A4tnTgQw1zON7u2sZMU7SzOeVH4tj20++9ggL+V6FDOFMTaFFYA==
 
+caniuse-lite@^1.0.30001669:
+  version "1.0.30001687"
+  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001687.tgz#d0ac634d043648498eedf7a3932836beba90ebae"
+  integrity sha512-0S/FDhf4ZiqrTUiQ39dKeUjYRjkv7lOZU1Dgif2rIqrTzX/1wV2hfKu9TOm1IHkdSijfLswxTFzl/cvir+SLSQ==
+
 capital-case@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669"
@@ -4663,6 +5560,17 @@ chai@^4.3.10:
     pathval "^1.1.1"
     type-detect "^4.0.8"
 
+chai@^5.1.1:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/chai/-/chai-5.1.2.tgz#3afbc340b994ae3610ca519a6c70ace77ad4378d"
+  integrity sha512-aGtmf24DW6MLHHG5gCx4zaI3uBq3KRtxeVs0DjFH6Z0rDNbsvTxFASFvdj79pxjxZ8/5u3PIiN3IwEIQkiiuPw==
+  dependencies:
+    assertion-error "^2.0.1"
+    check-error "^2.1.1"
+    deep-eql "^5.0.1"
+    loupe "^3.1.0"
+    pathval "^2.0.0"
+
 chalk@^2.0.0, chalk@^2.4.2:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@@ -4770,6 +5678,11 @@ check-error@^1.0.3:
   dependencies:
     get-func-name "^2.0.2"
 
+check-error@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/check-error/-/check-error-2.1.1.tgz#87eb876ae71ee388fa0471fe423f494be1d96ccc"
+  integrity sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==
+
 chokidar@^3.5.3:
   version "3.5.3"
   resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
@@ -4862,6 +5775,11 @@ clsx@^1.2.1:
   resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.2.1.tgz#0ddc4a20a549b59c93a4116bb26f5294ca17dc12"
   integrity sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg==
 
+clsx@^2.1.0, clsx@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/clsx/-/clsx-2.1.1.tgz#eed397c9fd8bd882bfb18deab7102049a2f32999"
+  integrity sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==
+
 codemirror@^5.62.0:
   version "5.65.10"
   resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-5.65.10.tgz#4276a93b8534ce91f14b733ba9a1ac949666eac9"
@@ -5043,9 +5961,9 @@ cross-inspect@1.0.0:
     tslib "^2.4.0"
 
 cross-spawn@^7.0.2:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
+  integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
   dependencies:
     path-key "^3.1.0"
     shebang-command "^2.0.0"
@@ -5097,6 +6015,11 @@ csstype@^3.0.2, csstype@^3.0.6, csstype@^3.0.7, csstype@^3.1.0, csstype@^3.1.1:
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.1.tgz#841b532c45c758ee546a11d5bd7b7b473c8c30b9"
   integrity sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==
 
+csstype@^3.1.3:
+  version "3.1.3"
+  resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81"
+  integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==
+
 "d3-array@2 - 3", "d3-array@2.10.0 - 3":
   version "3.2.4"
   resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.4.tgz#15fec33b237f97ac5d7c986dc77da273a8ed0bb5"
@@ -5267,6 +6190,11 @@ deep-eql@^4.1.3:
   dependencies:
     type-detect "^4.0.0"
 
+deep-eql@^5.0.1:
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-5.0.2.tgz#4b756d8d770a9257300825d52a2c2cff99c3a341"
+  integrity sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==
+
 deep-equal@^2.0.5:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.2.2.tgz#9b2635da569a13ba8e1cc159c2f744071b115daa"
@@ -5317,6 +6245,20 @@ define-data-property@^1.0.1, define-data-property@^1.1.1:
     gopd "^1.0.1"
     has-property-descriptors "^1.0.0"
 
+define-data-property@^1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e"
+  integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==
+  dependencies:
+    es-define-property "^1.0.0"
+    es-errors "^1.3.0"
+    gopd "^1.0.1"
+
+define-lazy-prop@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz#3f7ae421129bcaaac9bc74905c98a0009ec9ee7f"
+  integrity sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==
+
 define-properties@^1.1.3, define-properties@^1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.4.tgz#0b14d7bd7fbeb2f3572c3a7eda80ea5d57fb05b1"
@@ -5344,7 +6286,7 @@ dependency-graph@^0.11.0:
   resolved "https://registry.yarnpkg.com/dependency-graph/-/dependency-graph-0.11.0.tgz#ac0ce7ed68a54da22165a85e97a01d53f5eb2e27"
   integrity sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==
 
-dequal@^2.0.3:
+dequal@^2.0.2, dequal@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.3.tgz#2644214f1997d39ed0ee0ece72335490a7ac67be"
   integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==
@@ -5405,6 +6347,11 @@ dom-accessibility-api@^0.5.6, dom-accessibility-api@^0.5.9:
   resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz#5a7429e6066eb3664d911e33fb0e45de8eb08453"
   integrity sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==
 
+dom-accessibility-api@^0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz#993e925cc1d73f2c662e7d75dd5a5445259a8fd8"
+  integrity sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==
+
 dom-align@^1.7.0:
   version "1.12.2"
   resolved "https://registry.yarnpkg.com/dom-align/-/dom-align-1.12.2.tgz#0f8164ebd0c9c21b0c790310493cd855892acd4b"
@@ -5485,6 +6432,11 @@ electron-to-chromium@^1.4.535:
   resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.576.tgz#0c6940fdc0d60f7e34bd742b29d8fa847c9294d1"
   integrity sha512-yXsZyXJfAqzWk1WKryr0Wl0MN2D47xodPvEEwlVePBnhU5E7raevLQR+E6b9JAD3GfL/7MbAL9ZtWQQPcLx7wA==
 
+electron-to-chromium@^1.5.41:
+  version "1.5.71"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.71.tgz#d8b5dba1e55b320f2f4e9b1ca80738f53fcfec2b"
+  integrity sha512-dB68l59BI75W1BUGVTAEJy45CEVuEGy9qPVVQ8pnHyHMn36PLPPoE1mjLH+lo9rKulO3HC2OhbACI/8tCqJBcA==
+
 emoji-regex@^8.0.0:
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
@@ -5576,6 +6528,16 @@ es-abstract@^1.19.0, es-abstract@^1.20.4:
     string.prototype.trimstart "^1.0.5"
     unbox-primitive "^1.0.2"
 
+es-define-property@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa"
+  integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==
+
+es-errors@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f"
+  integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==
+
 es-get-iterator@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/es-get-iterator/-/es-get-iterator-1.1.3.tgz#3ef87523c5d464d41084b2c3c9c214f1199763d6"
@@ -5607,6 +6569,43 @@ es-to-primitive@^1.2.1:
     is-date-object "^1.0.1"
     is-symbol "^1.0.2"
 
+esbuild-register@^3.5.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/esbuild-register/-/esbuild-register-3.6.0.tgz#cf270cfa677baebbc0010ac024b823cbf723a36d"
+  integrity sha512-H2/S7Pm8a9CL1uhp9OvjwrBh5Pvx0H8qVOxNu8Wed9Y7qv56MPtq+GGM8RJpq6glYJn9Wspr8uw7l55uyinNeg==
+  dependencies:
+    debug "^4.3.4"
+
+"esbuild@^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0":
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.24.0.tgz#f2d470596885fcb2e91c21eb3da3b3c89c0b55e7"
+  integrity sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ==
+  optionalDependencies:
+    "@esbuild/aix-ppc64" "0.24.0"
+    "@esbuild/android-arm" "0.24.0"
+    "@esbuild/android-arm64" "0.24.0"
+    "@esbuild/android-x64" "0.24.0"
+    "@esbuild/darwin-arm64" "0.24.0"
+    "@esbuild/darwin-x64" "0.24.0"
+    "@esbuild/freebsd-arm64" "0.24.0"
+    "@esbuild/freebsd-x64" "0.24.0"
+    "@esbuild/linux-arm" "0.24.0"
+    "@esbuild/linux-arm64" "0.24.0"
+    "@esbuild/linux-ia32" "0.24.0"
+    "@esbuild/linux-loong64" "0.24.0"
+    "@esbuild/linux-mips64el" "0.24.0"
+    "@esbuild/linux-ppc64" "0.24.0"
+    "@esbuild/linux-riscv64" "0.24.0"
+    "@esbuild/linux-s390x" "0.24.0"
+    "@esbuild/linux-x64" "0.24.0"
+    "@esbuild/netbsd-x64" "0.24.0"
+    "@esbuild/openbsd-arm64" "0.24.0"
+    "@esbuild/openbsd-x64" "0.24.0"
+    "@esbuild/sunos-x64" "0.24.0"
+    "@esbuild/win32-arm64" "0.24.0"
+    "@esbuild/win32-ia32" "0.24.0"
+    "@esbuild/win32-x64" "0.24.0"
+
 esbuild@^0.18.10:
   version "0.18.20"
   resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.18.20.tgz#4709f5a34801b43b799ab7d6d82f7284a9b7a7a6"
@@ -5640,6 +6639,11 @@ escalade@^3.1.1:
   resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
   integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==
 
+escalade@^3.2.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5"
+  integrity sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==
+
 escape-html@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
@@ -5858,6 +6862,11 @@ espree@^9.5.2:
     acorn-jsx "^5.3.2"
     eslint-visitor-keys "^3.4.1"
 
+esprima@~4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"
+  integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==
+
 esquery@^1.4.2:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.5.0.tgz#6ce17738de8577694edd7361c57182ac8cb0db0b"
@@ -5887,6 +6896,13 @@ estree-walker@^2.0.2:
   resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-2.0.2.tgz#52f010178c2a4c117a7757cfe942adb7d2da4cac"
   integrity sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==
 
+estree-walker@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-3.0.3.tgz#67c3e549ec402a487b4fc193d1953a524752340d"
+  integrity sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==
+  dependencies:
+    "@types/estree" "^1.0.0"
+
 esutils@^2.0.2:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
@@ -6221,6 +7237,17 @@ get-intrinsic@^1.2.0, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2:
     has-symbols "^1.0.3"
     hasown "^2.0.0"
 
+get-intrinsic@^1.2.4:
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd"
+  integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==
+  dependencies:
+    es-errors "^1.3.0"
+    function-bind "^1.1.2"
+    has-proto "^1.0.1"
+    has-symbols "^1.0.3"
+    hasown "^2.0.0"
+
 get-symbol-description@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.0.tgz#7fdb81c900101fbd564dd5f1a30af5aadc1e58d6"
@@ -6387,6 +7414,13 @@ has-property-descriptors@^1.0.0:
   dependencies:
     get-intrinsic "^1.1.1"
 
+has-property-descriptors@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854"
+  integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==
+  dependencies:
+    es-define-property "^1.0.0"
+
 has-proto@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.1.tgz#1885c1305538958aff469fef37937c22795408e0"
@@ -6404,6 +7438,13 @@ has-tostringtag@^1.0.0:
   dependencies:
     has-symbols "^1.0.2"
 
+has-tostringtag@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc"
+  integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==
+  dependencies:
+    has-symbols "^1.0.3"
+
 has@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796"
@@ -6418,6 +7459,13 @@ hasown@^2.0.0:
   dependencies:
     function-bind "^1.1.2"
 
+hasown@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003"
+  integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==
+  dependencies:
+    function-bind "^1.1.2"
+
 hast-to-hyperscript@^9.0.0:
   version "9.0.1"
   resolved "https://registry.yarnpkg.com/hast-to-hyperscript/-/hast-to-hyperscript-9.0.1.tgz#9b67fd188e4c81e8ad66f803855334173920218d"
@@ -6769,7 +7817,7 @@ is-alphanumerical@^1.0.0:
     is-alphabetical "^1.0.0"
     is-decimal "^1.0.0"
 
-is-arguments@^1.1.1:
+is-arguments@^1.0.4, is-arguments@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b"
   integrity sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==
@@ -6827,6 +7875,13 @@ is-core-module@^2.11.0, is-core-module@^2.9.0:
   dependencies:
     has "^1.0.3"
 
+is-core-module@^2.13.0:
+  version "2.15.1"
+  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37"
+  integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==
+  dependencies:
+    hasown "^2.0.2"
+
 is-date-object@^1.0.1:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.4.tgz#550cfcc03afada05eea3dd30981c7b09551f73e5"
@@ -6844,7 +7899,7 @@ is-decimal@^1.0.0:
   resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-1.0.4.tgz#65a3a5958a1c5b63a706e1b333d7cd9f630d3fa5"
   integrity sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==
 
-is-docker@^2.0.0:
+is-docker@^2.0.0, is-docker@^2.1.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa"
   integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==
@@ -6878,6 +7933,13 @@ is-fullwidth-code-point@^3.0.0:
   resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
   integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
 
+is-generator-function@^1.0.7:
+  version "1.0.10"
+  resolved "https://registry.yarnpkg.com/is-generator-function/-/is-generator-function-1.0.10.tgz#f1558baf1ac17e0deea7c0415c438351ff2b3c72"
+  integrity sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==
+  dependencies:
+    has-tostringtag "^1.0.0"
+
 is-glob@4.0.3, is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
@@ -6999,6 +8061,13 @@ is-typed-array@^1.1.10:
   dependencies:
     which-typed-array "^1.1.11"
 
+is-typed-array@^1.1.3:
+  version "1.1.13"
+  resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.13.tgz#d6c5ca56df62334959322d7d7dd1cca50debe229"
+  integrity sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==
+  dependencies:
+    which-typed-array "^1.1.14"
+
 is-unc-path@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-unc-path/-/is-unc-path-1.0.0.tgz#d731e8898ed090a12c352ad2eaed5095ad322c9d"
@@ -7048,7 +8117,7 @@ is-windows@^1.0.1:
   resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d"
   integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==
 
-is-wsl@^2.1.1:
+is-wsl@^2.1.1, is-wsl@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271"
   integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==
@@ -7127,6 +8196,11 @@ js-yaml@^4.0.0, js-yaml@^4.1.0:
   dependencies:
     argparse "^2.0.1"
 
+jsdoc-type-pratt-parser@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/jsdoc-type-pratt-parser/-/jsdoc-type-pratt-parser-4.1.0.tgz#ff6b4a3f339c34a6c188cbf50a16087858d22113"
+  integrity sha512-Hicd6JK5Njt2QB6XYFS7ok9e37O8AYk3jTcppG4YVQnYjOemymvTcmc7OWsmq/Qqj5TdRFO5/x/tIPmBeRtGHg==
+
 jsdom@^22.1.0:
   version "22.1.0"
   resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-22.1.0.tgz#0fca6d1a37fbeb7f4aac93d1090d782c56b611c8"
@@ -7161,6 +8235,11 @@ jsesc@^2.5.1:
   resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4"
   integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==
 
+jsesc@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.0.2.tgz#bb8b09a6597ba426425f2e4a07245c3d00b9343e"
+  integrity sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==
+
 json-parse-even-better-errors@^2.3.0:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d"
@@ -7388,6 +8467,11 @@ loupe@^2.3.6:
   dependencies:
     get-func-name "^2.0.1"
 
+loupe@^3.1.0, loupe@^3.1.1, loupe@^3.1.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/loupe/-/loupe-3.1.2.tgz#c86e0696804a02218f2206124c45d8b15291a240"
+  integrity sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==
+
 lower-case-first@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/lower-case-first/-/lower-case-first-2.0.2.tgz#64c2324a2250bf7c37c5901e76a5b5309301160b"
@@ -7429,6 +8513,20 @@ lz-string@^1.5.0:
   resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.5.0.tgz#c1ab50f77887b712621201ba9fd4e3a6ed099941"
   integrity sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==
 
+magic-string@^0.27.0:
+  version "0.27.0"
+  resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.27.0.tgz#e4a3413b4bab6d98d2becffd48b4a257effdbbf3"
+  integrity sha512-8UnnX2PeRAPZuN12svgR9j7M1uWMovg/CEnIwIG0LFkXSJJe4PdfUGiTGl8V9bsBHFUtfVINcSyYxd7q+kx9fA==
+  dependencies:
+    "@jridgewell/sourcemap-codec" "^1.4.13"
+
+magic-string@^0.30.0:
+  version "0.30.14"
+  resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.30.14.tgz#e9bb29870b81cfc1ec3cc656552f5a7fcbf19077"
+  integrity sha512-5c99P1WKTed11ZC0HMJOj6CDIue6F8ySu+bJL+85q1zBEIY8IklrJ1eiKC2NDRh3Ct3FcvmJPyQHb9erXMTJNw==
+  dependencies:
+    "@jridgewell/sourcemap-codec" "^1.5.0"
+
 magic-string@^0.30.1:
   version "0.30.5"
   resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.30.5.tgz#1994d980bd1c8835dc6e78db7cbd4ae4f24746f9"
@@ -7459,6 +8557,11 @@ map-cache@^0.2.0:
   resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf"
   integrity sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=
 
+map-or-similar@^1.5.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/map-or-similar/-/map-or-similar-1.5.0.tgz#6de2653174adfb5d9edc33c69d3e92a1b76faf08"
+  integrity sha512-0aF7ZmVon1igznGI4VS30yugpduQW3y3GkcgGJOp7d8x8QrizhigUxjI/m2UojsXXto+jLAH3KSz+xOJTiORjg==
+
 markdown-table@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/markdown-table/-/markdown-table-2.0.0.tgz#194a90ced26d31fe753d8b9434430214c011865b"
@@ -7599,6 +8702,13 @@ mdurl@^1.0.0:
   resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e"
   integrity sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=
 
+memoizerific@^1.11.3:
+  version "1.11.3"
+  resolved "https://registry.yarnpkg.com/memoizerific/-/memoizerific-1.11.3.tgz#7c87a4646444c32d75438570905f2dbd1b1a805a"
+  integrity sha512-/EuHYwAPdLtXwAwSZkh/Gutery6pD2KYd44oQLhAvQp/50mpyduZh8Q7PYHXTCJ+wuXxt7oij2LXyIJOOYFPog==
+  dependencies:
+    map-or-similar "^1.5.0"
+
 merge2@^1.3.0, merge2@^1.4.1:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
@@ -7704,7 +8814,7 @@ min-document@^2.19.0:
   dependencies:
     dom-walk "^0.1.0"
 
-min-indent@^1.0.0:
+min-indent@^1.0.0, min-indent@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869"
   integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==
@@ -7811,9 +8921,9 @@ nanoevents@^5.1.13:
   integrity sha512-JFAeG9fp0QZnRoESHjkbVFbZ9BkOXkkagUVwZVo/pkSX+Fq1VKlY+5og/8X9CYc6C7vje/CV+bwJ5M2X0+IY9Q==
 
 nanoid@^3.3.6:
-  version "3.3.6"
-  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c"
-  integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==
+  version "3.3.8"
+  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf"
+  integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==
 
 natural-compare-lite@^1.4.0:
   version "1.4.0"
@@ -7863,6 +8973,11 @@ node-releases@^2.0.13:
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.13.tgz#d5ed1627c23e3461e819b02e57b75e4899b1c81d"
   integrity sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==
 
+node-releases@^2.0.18:
+  version "2.0.18"
+  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.18.tgz#f010e8d35e2fe8d6b2944f03f70213ecedc4ca3f"
+  integrity sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==
+
 normalize-path@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9"
@@ -8001,6 +9116,15 @@ open@^7.3.1:
     is-docker "^2.0.0"
     is-wsl "^2.1.1"
 
+open@^8.0.4:
+  version "8.4.2"
+  resolved "https://registry.yarnpkg.com/open/-/open-8.4.2.tgz#5b5ffe2a8f793dcd2aad73e550cb87b59cb084f9"
+  integrity sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==
+  dependencies:
+    define-lazy-prop "^2.0.0"
+    is-docker "^2.1.1"
+    is-wsl "^2.2.0"
+
 optimism@^0.16.0:
   version "0.16.1"
   resolved "https://registry.yarnpkg.com/optimism/-/optimism-0.16.1.tgz#7c8efc1f3179f18307b887e18c15c5b7133f6e7d"
@@ -8236,16 +9360,31 @@ pathval@^1.1.1:
   resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.1.tgz#8534e77a77ce7ac5a2512ea21e0fdb8fcf6c3d8d"
   integrity sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==
 
+pathval@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/pathval/-/pathval-2.0.0.tgz#7e2550b422601d4f6b8e26f1301bc8f15a741a25"
+  integrity sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==
+
 picocolors@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
   integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==
 
+picocolors@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b"
+  integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==
+
 picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
   integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
 
+picomatch@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.2.tgz#77c742931e8f3b8820946c76cd0c1f13730d1dab"
+  integrity sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==
+
 pify@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231"
@@ -8260,6 +9399,18 @@ pkg-types@^1.0.3:
     mlly "^1.2.0"
     pathe "^1.1.0"
 
+polished@^4.2.2:
+  version "4.3.1"
+  resolved "https://registry.yarnpkg.com/polished/-/polished-4.3.1.tgz#5a00ae32715609f83d89f6f31d0f0261c6170548"
+  integrity sha512-OBatVyC/N7SCW/FaDHrSd+vn0o5cS855TOmYi4OkdWUMSJCET/xip//ch8xGUvtr3i44X9LVyWwQlRMTN3pwSA==
+  dependencies:
+    "@babel/runtime" "^7.17.8"
+
+possible-typed-array-names@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f"
+  integrity sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==
+
 postcss-value-parser@^4.0.2:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514"
@@ -8315,6 +9466,11 @@ prismjs@^1.22.0, prismjs@^1.27.0, prismjs@~1.23.0:
   resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.29.0.tgz#f113555a8fa9b57c35e637bba27509dcf802dd12"
   integrity sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==
 
+process@^0.11.10:
+  version "0.11.10"
+  resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
+  integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==
+
 promise@^7.1.1:
   version "7.3.1"
   resolved "https://registry.yarnpkg.com/promise/-/promise-7.3.1.tgz#064b72602b18f90f29192b8b1bc418ffd1ebd3bf"
@@ -8331,7 +9487,7 @@ prop-types@15.7.2:
     object-assign "^4.1.1"
     react-is "^16.8.1"
 
-prop-types@^15.5.10, prop-types@^15.6.0, prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1:
+prop-types@^15.5.10, prop-types@^15.5.7, prop-types@^15.6.0, prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1:
   version "15.8.1"
   resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5"
   integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==
@@ -8968,6 +10124,42 @@ react-color@^2.19.3:
     reactcss "^1.2.0"
     tinycolor2 "^1.4.1"
 
+react-confetti@^6.1.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/react-confetti/-/react-confetti-6.1.0.tgz#03dc4340d955acd10b174dbf301f374a06e29ce6"
+  integrity sha512-7Ypx4vz0+g8ECVxr88W9zhcQpbeujJAVqL14ZnXJ3I23mOI9/oBVTQ3dkJhUmB0D6XOtCZEM6N0Gm9PMngkORw==
+  dependencies:
+    tween-functions "^1.2.0"
+
+react-docgen-typescript@^2.2.2:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/react-docgen-typescript/-/react-docgen-typescript-2.2.2.tgz#4611055e569edc071204aadb20e1c93e1ab1659c"
+  integrity sha512-tvg2ZtOpOi6QDwsb3GZhOjDkkX0h8Z2gipvTg6OVMUyoYoURhEiRNePT8NZItTVCDh39JJHnLdfCOkzoLbFnTg==
+
+react-docgen@^7.0.0:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/react-docgen/-/react-docgen-7.1.0.tgz#4b41e557dab939a5157be09ee532fd09c07d99fc"
+  integrity sha512-APPU8HB2uZnpl6Vt/+0AFoVYgSRtfiP6FLrZgPPTDmqSb2R4qZRbgd0A3VzIFxDt5e+Fozjx79WjLWnF69DK8g==
+  dependencies:
+    "@babel/core" "^7.18.9"
+    "@babel/traverse" "^7.18.9"
+    "@babel/types" "^7.18.9"
+    "@types/babel__core" "^7.18.0"
+    "@types/babel__traverse" "^7.18.0"
+    "@types/doctrine" "^0.0.9"
+    "@types/resolve" "^1.20.2"
+    doctrine "^3.0.0"
+    resolve "^1.22.1"
+    strip-indent "^4.0.0"
+
+"react-dom@^16.8.0 || ^17.0.0 || ^18.0.0":
+  version "18.3.1"
+  resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.3.1.tgz#c2265d79511b57d479b3dd3fdfa51536494c5cb4"
+  integrity sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==
+  dependencies:
+    loose-envify "^1.1.0"
+    scheduler "^0.23.2"
+
 react-dom@^17.0.0:
   version "17.0.2"
   resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-17.0.2.tgz#ecffb6845e3ad8dbfcdc498f0d0a939736502c23"
@@ -9040,6 +10232,11 @@ react-is@^18.0.0, react-is@^18.2.0:
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b"
   integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==
 
+react-is@^18.3.1:
+  version "18.3.1"
+  resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.3.1.tgz#e83557dc12eae63a99e003a46388b1dcbb44db7e"
+  integrity sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==
+
 react-js-cron@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/react-js-cron/-/react-js-cron-2.1.0.tgz#ce88a3260617222d8e1dc51534bb6606088304fc"
@@ -9157,6 +10354,13 @@ react-visibility-sensor@^5.1.1:
   dependencies:
     prop-types "^15.7.2"
 
+"react@^16.8.0 || ^17.0.0 || ^18.0.0":
+  version "18.3.1"
+  resolved "https://registry.yarnpkg.com/react/-/react-18.3.1.tgz#49ab892009c53933625bd16b2533fc754cab2891"
+  integrity sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==
+  dependencies:
+    loose-envify "^1.1.0"
+
 react@^17.0.0:
   version "17.0.2"
   resolved "https://registry.yarnpkg.com/react/-/react-17.0.2.tgz#d0b5cc516d29eb3eee383f75b62864cfb6800037"
@@ -9202,6 +10406,17 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
+recast@^0.23.5:
+  version "0.23.9"
+  resolved "https://registry.yarnpkg.com/recast/-/recast-0.23.9.tgz#587c5d3a77c2cfcb0c18ccce6da4361528c2587b"
+  integrity sha512-Hx/BGIbwj+Des3+xy5uAtAbdCyqK9y9wbBcDFDYanLS9JnMqf7OeF87HQwUimE87OEc72mr6tkKUKMBBL+hF9Q==
+  dependencies:
+    ast-types "^0.16.1"
+    esprima "~4.0.0"
+    source-map "~0.6.1"
+    tiny-invariant "^1.3.3"
+    tslib "^2.0.1"
+
 redent@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f"
@@ -9240,6 +10455,11 @@ regenerator-runtime@^0.13.11, regenerator-runtime@^0.13.4:
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9"
   integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==
 
+regenerator-runtime@^0.14.0:
+  version "0.14.1"
+  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f"
+  integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==
+
 regexp.prototype.flags@^1.4.3:
   version "1.4.3"
   resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz#87cab30f80f66660181a3bb7bf5981a872b367ac"
@@ -9462,6 +10682,15 @@ resolve@^1.12.0, resolve@^1.19.0, resolve@^1.22.1:
     path-parse "^1.0.7"
     supports-preserve-symlinks-flag "^1.0.0"
 
+resolve@^1.22.8:
+  version "1.22.8"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"
+  integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==
+  dependencies:
+    is-core-module "^2.13.0"
+    path-parse "^1.0.7"
+    supports-preserve-symlinks-flag "^1.0.0"
+
 resolve@^2.0.0-next.4:
   version "2.0.0-next.4"
   resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.4.tgz#3d37a113d6429f496ec4752d2a2e58efb1fd4660"
@@ -9606,6 +10835,13 @@ scheduler@^0.20.2:
     loose-envify "^1.1.0"
     object-assign "^4.1.1"
 
+scheduler@^0.23.2:
+  version "0.23.2"
+  resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.23.2.tgz#414ba64a3b282892e944cf2108ecc078d115cdc3"
+  integrity sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==
+  dependencies:
+    loose-envify "^1.1.0"
+
 screenfull@^5.1.0:
   version "5.2.0"
   resolved "https://registry.yarnpkg.com/screenfull/-/screenfull-5.2.0.tgz#6533d524d30621fc1283b9692146f3f13a93d1ba"
@@ -9662,6 +10898,11 @@ semver@^7.5.4:
   dependencies:
     lru-cache "^6.0.0"
 
+semver@^7.6.2:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
 sentence-case@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/sentence-case/-/sentence-case-3.0.4.tgz#3645a7b8c117c787fde8702056225bb62a45131f"
@@ -9686,6 +10927,18 @@ set-function-length@^1.1.1:
     gopd "^1.0.1"
     has-property-descriptors "^1.0.0"
 
+set-function-length@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449"
+  integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==
+  dependencies:
+    define-data-property "^1.1.4"
+    es-errors "^1.3.0"
+    function-bind "^1.1.2"
+    get-intrinsic "^1.2.4"
+    gopd "^1.0.1"
+    has-property-descriptors "^1.0.2"
+
 set-function-name@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.1.tgz#12ce38b7954310b9f61faa12701620a0c882793a"
@@ -9815,7 +11068,7 @@ source-map@^0.5.7:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
   integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
 
-source-map@^0.6.1, source-map@~0.6.0:
+source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
@@ -9903,6 +11156,13 @@ stop-iteration-iterator@^1.0.0:
   dependencies:
     internal-slot "^1.0.4"
 
+storybook@^8.1.11:
+  version "8.4.7"
+  resolved "https://registry.yarnpkg.com/storybook/-/storybook-8.4.7.tgz#a3068787a58074cec1b4197eed1c4427ec644b3f"
+  integrity sha512-RP/nMJxiWyFc8EVMH5gp20ID032Wvk+Yr3lmKidoegto5Iy+2dVQnUoElZb2zpbVXNHWakGuAkfI0dY1Hfp/vw==
+  dependencies:
+    "@storybook/core" "8.4.7"
+
 streamsearch@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"
@@ -10008,6 +11268,13 @@ strip-indent@^3.0.0:
   dependencies:
     min-indent "^1.0.0"
 
+strip-indent@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-4.0.0.tgz#b41379433dd06f5eae805e21d631e07ee670d853"
+  integrity sha512-mnVSV2l+Zv6BLpSD/8V87CW/y9EmmbYzGCIavsnsI6/nwn26DwffM/yztm30Z/I2DY9wdS3vXVCMnHDgZaVNoA==
+  dependencies:
+    min-indent "^1.0.1"
+
 strip-json-comments@^3.1.0, strip-json-comments@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"
@@ -10048,6 +11315,11 @@ stylis@4.1.3, stylis@^4.0.6:
   resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.1.3.tgz#fd2fbe79f5fed17c55269e16ed8da14c84d069f7"
   integrity sha512-GP6WDNWf+o403jrEp9c5jibKavrtLW+/qYGhFxFrG8maXhwTBI7gLLhiBb0o7uFccWN+EOS9aMO6cGHWAO07OA==
 
+stylis@4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.2.0.tgz#79daee0208964c8fe695a42fcffcac633a211a51"
+  integrity sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==
+
 supports-color@^5.3.0, supports-color@^5.5.0:
   version "5.5.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f"
@@ -10129,6 +11401,11 @@ tiny-invariant@^1.0.2:
   resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.1.0.tgz#634c5f8efdc27714b7f386c35e6760991d230875"
   integrity sha512-ytxQvrb1cPc9WBEI/HSeYYoGD0kWnGEOR8RY6KomWLBVhqz0RgTwVO9dLrGz7dC+nN9llyI7OKAgRq8Vq4ZBSw==
 
+tiny-invariant@^1.3.1, tiny-invariant@^1.3.3:
+  version "1.3.3"
+  resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.3.tgz#46680b7a873a0d5d10005995eb90a70d74d60127"
+  integrity sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==
+
 tiny-warning@^1.0.0, tiny-warning@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/tiny-warning/-/tiny-warning-1.0.3.tgz#94a30db453df4c643d0fd566060d60a875d84754"
@@ -10149,11 +11426,21 @@ tinypool@^0.7.0:
   resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-0.7.0.tgz#88053cc99b4a594382af23190c609d93fddf8021"
   integrity sha512-zSYNUlYSMhJ6Zdou4cJwo/p7w5nmAH17GRfU/ui3ctvjXFErXXkruT4MWW6poDeXgCaIBlGLrfU6TbTXxyGMww==
 
+tinyrainbow@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/tinyrainbow/-/tinyrainbow-1.2.0.tgz#5c57d2fc0fb3d1afd78465c33ca885d04f02abb5"
+  integrity sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==
+
 tinyspy@^2.1.1:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/tinyspy/-/tinyspy-2.2.0.tgz#9dc04b072746520b432f77ea2c2d17933de5d6ce"
   integrity sha512-d2eda04AN/cPOR89F7Xv5bK/jrQEhmcLFe6HFldoeO9AJtps+fqEnh486vnT/8y4bw38pSyxDcTCAq+Ks2aJTg==
 
+tinyspy@^3.0.0:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/tinyspy/-/tinyspy-3.0.2.tgz#86dd3cf3d737b15adcf17d7887c84a75201df20a"
+  integrity sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==
+
 title-case@^3.0.3:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/title-case/-/title-case-3.0.3.tgz#bc689b46f02e411f1d1e1d081f7c3deca0489982"
@@ -10217,6 +11504,11 @@ ts-api-utils@^1.0.1:
   resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.0.3.tgz#f12c1c781d04427313dbac808f453f050e54a331"
   integrity sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==
 
+ts-dedent@^2.0.0, ts-dedent@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-2.2.0.tgz#39e4bd297cd036292ae2394eb3412be63f563bb5"
+  integrity sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==
+
 ts-easing@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/ts-easing/-/ts-easing-0.2.0.tgz#c8a8a35025105566588d87dbda05dd7fbfa5a4ec"
@@ -10244,6 +11536,15 @@ tsconfig-paths@^3.14.1:
     minimist "^1.2.6"
     strip-bom "^3.0.0"
 
+tsconfig-paths@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz#ef78e19039133446d244beac0fd6a1632e2d107c"
+  integrity sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==
+  dependencies:
+    json5 "^2.2.2"
+    minimist "^1.2.6"
+    strip-bom "^3.0.0"
+
 tslib@^1.10.0, tslib@^1.8.1:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
@@ -10254,6 +11555,11 @@ tslib@^2.0.0, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.4.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e"
   integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==
 
+tslib@^2.0.1:
+  version "2.8.1"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f"
+  integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==
+
 tslib@^2.3.1, tslib@^2.5.0, tslib@^2.6.1, tslib@^2.6.2:
   version "2.6.2"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
@@ -10298,6 +11604,11 @@ turndown@^7.1.1:
   dependencies:
     domino "^2.1.6"
 
+tween-functions@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/tween-functions/-/tween-functions-1.2.0.tgz#1ae3a50e7c60bb3def774eac707acbca73bbc3ff"
+  integrity sha512-PZBtLYcCLtEcjL14Fzb1gSxPBeL7nWvGhO5ZFPGqziCcr8uvHp0NDmdjBchp6KHL+tExcg0m3NISmKxhU394dA==
+
 type-check@^0.4.0, type-check@~0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1"
@@ -10325,7 +11636,7 @@ type-fest@^1.2.0:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-1.4.0.tgz#e9fb813fe3bf1744ec359d55d1affefa76f14be1"
   integrity sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==
 
-type-fest@^2.0.0:
+type-fest@^2.0.0, type-fest@^2.19.0:
   version "2.19.0"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-2.19.0.tgz#88068015bb33036a598b952e55e9311a60fd3a9b"
   integrity sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==
@@ -10438,6 +11749,14 @@ unixify@^1.0.0:
   dependencies:
     normalize-path "^2.1.1"
 
+unplugin@^1.3.1:
+  version "1.16.0"
+  resolved "https://registry.yarnpkg.com/unplugin/-/unplugin-1.16.0.tgz#ca0f248bf8798cd752dd02e5b381223b737cef72"
+  integrity sha512-5liCNPuJW8dqh3+DM6uNM2EI3MLLpCKp/KY+9pB5M2S2SR2qvvDHhKgBOaTWEbZTAws3CXfB0rKTIolWKL05VQ==
+  dependencies:
+    acorn "^8.14.0"
+    webpack-virtual-modules "^0.6.2"
+
 update-browserslist-db@^1.0.11:
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz#9a2a641ad2907ae7b3616506f4b977851db5b940"
@@ -10454,6 +11773,14 @@ update-browserslist-db@^1.0.13:
     escalade "^3.1.1"
     picocolors "^1.0.0"
 
+update-browserslist-db@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz#80846fba1d79e82547fb661f8d141e0945755fe5"
+  integrity sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==
+  dependencies:
+    escalade "^3.2.0"
+    picocolors "^1.1.0"
+
 upper-case-first@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/upper-case-first/-/upper-case-first-2.0.2.tgz#992c3273f882abd19d1e02894cc147117f844324"
@@ -10525,11 +11852,27 @@ util-deprecate@^1.0.1:
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
 
+util@^0.12.5:
+  version "0.12.5"
+  resolved "https://registry.yarnpkg.com/util/-/util-0.12.5.tgz#5f17a6059b73db61a875668781a1c2b136bd6fbc"
+  integrity sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==
+  dependencies:
+    inherits "^2.0.3"
+    is-arguments "^1.0.4"
+    is-generator-function "^1.0.7"
+    is-typed-array "^1.1.3"
+    which-typed-array "^1.1.2"
+
 uuid@^8.3.2:
   version "8.3.2"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
 
+uuid@^9.0.0:
+  version "9.0.1"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30"
+  integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==
+
 value-equal@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/value-equal/-/value-equal-1.0.1.tgz#1e0b794c734c5c0cade179c437d356d931a34d6c"
@@ -10706,6 +12049,11 @@ webidl-conversions@^7.0.0:
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a"
   integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==
 
+webpack-virtual-modules@^0.6.2:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz#057faa9065c8acf48f24cb57ac0e77739ab9a7e8"
+  integrity sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==
+
 whatwg-encoding@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz#e7635f597fd87020858626805a2729fa7698ac53"
@@ -10771,6 +12119,17 @@ which-typed-array@^1.1.11, which-typed-array@^1.1.9:
     gopd "^1.0.1"
     has-tostringtag "^1.0.0"
 
+which-typed-array@^1.1.14, which-typed-array@^1.1.2:
+  version "1.1.16"
+  resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.16.tgz#db4db429c4706feca2f01677a144278e4a8c216b"
+  integrity sha512-g+N+GAWiRj66DngFwHvISJd+ITsyphZvD1vChfVg6cEdnzy53GzB3oy0fUNlvhz7H7+MiqhYr26qxQShCpKTTQ==
+  dependencies:
+    available-typed-arrays "^1.0.7"
+    call-bind "^1.0.7"
+    for-each "^0.3.3"
+    gopd "^1.0.1"
+    has-tostringtag "^1.0.2"
+
 which@^2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
@@ -10824,6 +12183,11 @@ ws@^8.13.0:
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.13.0.tgz#9a9fb92f93cf41512a0735c8f4dd09b8a1211cd0"
   integrity sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==
 
+ws@^8.2.3:
+  version "8.18.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
+  integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
+
 xml-name-validator@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835"
diff --git a/docker/build.gradle b/docker/build.gradle
index c09bf16d1d724..7b36c0d9acdcf 100644
--- a/docker/build.gradle
+++ b/docker/build.gradle
@@ -8,6 +8,9 @@ import com.avast.gradle.dockercompose.tasks.ComposeDownForced
 apply from: "../gradle/versioning/versioning.gradle"
 
 ext {
+    compose_base = "profiles/docker-compose.yml"
+    project_name = "datahub"
+
     backend_profile_modules = [
             ':docker:elasticsearch-setup',
             ':docker:mysql-setup',
@@ -15,22 +18,131 @@ ext {
             ':datahub-upgrade',
             ':metadata-service:war',
     ]
-    quickstart_modules = backend_profile_modules + [
-            ':metadata-jobs:mce-consumer-job',
-            ':metadata-jobs:mae-consumer-job',
-            ':datahub-frontend'
+
+    python_services_modules = []
+
+    // Common configuration for all tasks
+    common_config = [
+            captureContainersOutput: true,
+            captureContainersOutputToFiles: project.file('build/container-logs')
     ]
 
-    debug_modules = quickstart_modules - [':metadata-jobs:mce-consumer-job',
-                                          ':metadata-jobs:mae-consumer-job']
-    compose_args = ['-f', 'profiles/docker-compose.yml']
-    debug_reloadable = [
-            'datahub-gms-debug',
-            'system-update-debug',
-            'frontend-debug'
+    // declarative task configuration
+    quickstart_configs = [
+            'quickstart': [
+                    profile: 'quickstart-consumers',
+                    modules: python_services_modules + backend_profile_modules + [
+                            ':datahub-frontend',
+                            ':metadata-jobs:mce-consumer-job',
+                            ':metadata-jobs:mae-consumer-job',
+                    ]
+            ],
+            'quickstartDebug': [
+                    profile: 'debug',
+                    modules: python_services_modules + backend_profile_modules + [':datahub-frontend'],
+                    isDebug: true
+            ],
+            'quickstartPg': [
+                    profile: 'quickstart-postgres',
+                    modules: (backend_profile_modules - [':docker:mysql-setup']) + [
+                            ':docker:postgres-setup',
+                            ':datahub-frontend'
+                    ]
+            ],
+            'quickstartPgDebug': [
+                    profile: 'debug-postgres',
+                    modules: python_services_modules + (backend_profile_modules - [':docker:mysql-setup']) + [
+                            ':docker:postgres-setup',
+                            ':datahub-frontend'
+                    ],
+                    isDebug: true
+            ],
+            'quickstartSlim': [
+                    profile: 'quickstart-backend',
+                    modules: backend_profile_modules + [':docker:datahub-ingestion'],
+                    additionalEnv: [
+                            'DATAHUB_ACTIONS_IMAGE': 'acryldata/datahub-ingestion',
+                            'ACTIONS_VERSION': "v${version}-slim",
+                            'ACTIONS_EXTRA_PACKAGES': 'acryl-datahub-actions[executor] acryl-datahub-actions',
+                            'ACTIONS_CONFIG': 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml',
+                            'DATAHUB_LOCAL_COMMON_ENV': "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
+                    ]
+            ],
+            'quickstartStorage': [
+                    profile: 'quickstart-storage',
+                    preserveVolumes: true
+            ]
     ]
-    // Postgres
-    pg_quickstart_modules = quickstart_modules - [':docker:mysql-setup'] + [':docker:postgres-setup']
+}
+
+// Register all quickstart tasks
+quickstart_configs.each { taskName, config ->
+    tasks.register(taskName)
+}
+
+// Dynamically create all quickstart tasks and configurations
+dockerCompose {
+    // Configure default settings that apply to all configurations
+    useComposeFiles = [compose_base]
+    projectName = project_name
+    projectNamePrefix = ''
+    buildBeforeUp = false
+    buildBeforePull = false
+    stopContainers = false
+    removeVolumes = false
+
+    quickstart_configs.each { taskName, config ->
+        "${taskName}" {
+            isRequiredBy(tasks.named(taskName))
+            if (config.profile) {
+                composeAdditionalArgs = ['--profile', config.profile]
+            }
+
+            // Common environment variables
+            environment.put 'DATAHUB_VERSION', config.isDebug ?
+                    System.getenv("DATAHUB_VERSION") ?: "v${version}" :
+                    "v${version}"
+            environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false'
+            environment.put "METADATA_TESTS_ENABLED", "true"
+            environment.put "DATAHUB_REPO", "${docker_registry}"
+
+            // Additional environment variables if specified
+            if (config.additionalEnv) {
+                config.additionalEnv.each { key, value ->
+                    environment.put key, value
+                }
+            }
+
+            useComposeFiles = [compose_base]
+            projectName = project_name
+            projectNamePrefix = ''
+            buildBeforeUp = false
+            buildBeforePull = false
+            stopContainers = false
+            removeVolumes = false
+
+            // Apply common configuration
+            common_config.each { key, value ->
+                delegate."${key}" = value
+            }
+
+            // Apply additional task-specific configuration if specified
+            if (config.additionalConfig) {
+                config.additionalConfig.each { key, value ->
+                    delegate."${key}" = value
+                }
+            }
+        }
+    }
+}
+
+// Configure dependencies for ComposeUp tasks
+quickstart_configs.each { taskName, config ->
+    if (config.modules) {
+        tasks.getByName("${taskName}ComposeUp").dependsOn(
+                config.modules.collect { it + ":${config.isDebug ? 'dockerTagDebug' : 'dockerTag'}" }
+        )
+    }
 }
 
 tasks.register('minDockerCompose2.20', Exec) {
@@ -38,18 +150,11 @@ tasks.register('minDockerCompose2.20', Exec) {
     args '-c', 'echo -e "$(docker compose version --short)\n2.20"|sort --version-sort --check=quiet --reverse'
 }
 
-tasks.register('quickstart') {}
-tasks.register('quickstartSlim') {}
-tasks.register('quickstartDebug') {}
-tasks.register('quickstartPg') {}
-tasks.register('quickstartStorage') {}
-
 tasks.register('quickstartNuke') {
     doFirst {
-        dockerCompose.quickstart.removeVolumes = true
-        dockerCompose.quickstartPg.removeVolumes = true
-        dockerCompose.quickstartSlim.removeVolumes = true
-        dockerCompose.quickstartDebug.removeVolumes = true
+        quickstart_configs.each { taskName, config ->
+            dockerCompose."${taskName}".removeVolumes = !config.preserveVolumes
+        }
     }
     finalizedBy(tasks.withType(ComposeDownForced))
 }
@@ -58,114 +163,17 @@ tasks.register('quickstartDown') {
     finalizedBy(tasks.withType(ComposeDownForced))
 }
 
-dockerCompose {
-    quickstart {
-        isRequiredBy(tasks.named('quickstart'))
-        composeAdditionalArgs = ['--profile', 'quickstart-consumers']
-
-        environment.put 'DATAHUB_VERSION', "v${version}"
-        environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
-
-        useComposeFiles = ['profiles/docker-compose.yml']
-        projectName = 'datahub'
-        projectNamePrefix = ''
-        buildBeforeUp = false
-        buildBeforePull = false
-        stopContainers = false
-        removeVolumes = false
-        captureContainersOutput = true
-        captureContainersOutputToFiles = project.file('build/container-logs')
-    }
-
-    quickstartPg {
-        isRequiredBy(tasks.named('quickstartPg'))
-        composeAdditionalArgs = ['--profile', 'quickstart-postgres']
-
-        environment.put 'DATAHUB_VERSION', "v${version}"
-        environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
-
-        useComposeFiles = ['profiles/docker-compose.yml']
-        projectName = 'datahub'
-        projectNamePrefix = ''
-        buildBeforeUp = false
-        buildBeforePull = false
-        stopContainers = false
-        removeVolumes = false
-    }
-
-    /**
-     * The smallest disk footprint required for Spark integration tests
-     *
-     * No frontend, mae, mce, or other services
-     */
-    quickstartSlim {
-        isRequiredBy(tasks.named('quickstartSlim'))
-        composeAdditionalArgs = ['--profile', 'quickstart-backend']
-
-        environment.put 'DATAHUB_VERSION', "v${version}"
-        environment.put "DATAHUB_ACTIONS_IMAGE", "acryldata/datahub-ingestion"
-        environment.put "ACTIONS_VERSION", "v${version}-slim"
-        environment.put "ACTIONS_EXTRA_PACKAGES", 'acryl-datahub-actions[executor] acryl-datahub-actions'
-        environment.put "ACTIONS_CONFIG", 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml'
-        environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
-        // disabled for spark-lineage smoke-test
-        environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
-
-        useComposeFiles = ['profiles/docker-compose.yml']
-        projectName = 'datahub'
-        projectNamePrefix = ''
-        buildBeforeUp = false
-        buildBeforePull = false
-        stopContainers = false
-        removeVolumes = false
-        captureContainersOutput = true
-        captureContainersOutputToFiles = project.file('build/container-logs')
-    }
-
-    quickstartDebug {
-        isRequiredBy(tasks.named('quickstartDebug'))
-        composeAdditionalArgs = ['--profile', 'debug']
-
-        environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
-
-        useComposeFiles = ['profiles/docker-compose.yml']
-        projectName = 'datahub'
-        projectNamePrefix = ''
-        buildBeforeUp = false
-        buildBeforePull = false
-        stopContainers = false
-        removeVolumes = false
-    }
-
-    quickstartStorage {
-        isRequiredBy(tasks.named('quickstartStorage'))
-        composeAdditionalArgs = ['--profile', 'quickstart-storage']
-
-        useComposeFiles = ['profiles/docker-compose.yml']
-        projectName = 'datahub'
-        projectNamePrefix = ''
-        buildBeforeUp = false
-        buildBeforePull = false
-        stopContainers = false
-        removeVolumes = false
-    }
-}
-tasks.getByName('quickstartComposeUp').dependsOn(
-        quickstart_modules.collect { it + ':dockerTag' })
-tasks.getByName('quickstartPgComposeUp').dependsOn(
-        pg_quickstart_modules.collect { it + ':dockerTag' })
-tasks.getByName('quickstartSlimComposeUp').dependsOn(
-        ([':docker:datahub-ingestion'] + backend_profile_modules)
-                .collect { it + ':dockerTag' })
-tasks.getByName('quickstartDebugComposeUp').dependsOn(
-        debug_modules.collect { it + ':dockerTagDebug' }
-)
 tasks.withType(ComposeUp).configureEach {
     shouldRunAfter('quickstartNuke')
     dependsOn tasks.named("minDockerCompose2.20")
 }
 
 task debugReload(type: Exec) {
-    def cmd = ['docker compose -p datahub --profile debug'] + compose_args + ['restart'] + debug_reloadable
+    def cmd = ['docker compose -p datahub --profile debug'] + ['-f', compose_base] + [
+            'restart',
+            'datahub-gms-debug',
+            'system-update-debug',
+            'frontend-debug'
+    ]
     commandLine 'bash', '-c', cmd.join(" ")
-}
+}
\ No newline at end of file
diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile
index b15bf3c6f9f17..47b10535f8dee 100644
--- a/docker/datahub-gms/Dockerfile
+++ b/docker/datahub-gms/Dockerfile
@@ -6,12 +6,12 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
 ARG GITHUB_REPO_URL=https://github.com
 ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
 
-FROM golang:1-alpine3.20 AS binary
+FROM golang:1-alpine3.21 AS binary
 
 # Re-declaring arg from above to make it available in this stage (will inherit default value)
 ARG ALPINE_REPO_URL
 
-ENV DOCKERIZE_VERSION=v0.6.1
+ENV DOCKERIZE_VERSION=v0.9.1
 WORKDIR /go/src/github.com/jwilder
 
 # Optionally set corporate mirror for apk
diff --git a/docker/datahub-ingestion-base/build.gradle b/docker/datahub-ingestion-base/build.gradle
index ef482de9256a3..f19faa227ca61 100644
--- a/docker/datahub-ingestion-base/build.gradle
+++ b/docker/datahub-ingestion-base/build.gradle
@@ -12,7 +12,7 @@ ext {
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
     docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
-    revision = 7 // increment to trigger rebuild
+    revision = 8 // increment to trigger rebuild
 }
 
 docker {
diff --git a/docker/datahub-ingestion/build.gradle b/docker/datahub-ingestion/build.gradle
index 113a6dcf0a1bd..b236a53c288f7 100644
--- a/docker/datahub-ingestion/build.gradle
+++ b/docker/datahub-ingestion/build.gradle
@@ -12,7 +12,7 @@ ext {
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
     docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
-    revision = 8 // increment to trigger rebuild
+    revision = 9 // increment to trigger rebuild
 }
 
 dependencies {
diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile
index 6edaa29ee1a8b..74375072761d8 100644
--- a/docker/datahub-mae-consumer/Dockerfile
+++ b/docker/datahub-mae-consumer/Dockerfile
@@ -6,12 +6,12 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
 ARG GITHUB_REPO_URL=https://github.com
 ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
 
-FROM golang:1-alpine3.20 AS binary
+FROM golang:1-alpine3.21 AS binary
 
 # Re-declaring arg from above to make it available in this stage (will inherit default value)
 ARG ALPINE_REPO_URL
 
-ENV DOCKERIZE_VERSION=v0.6.1
+ENV DOCKERIZE_VERSION=v0.9.1
 WORKDIR /go/src/github.com/jwilder
 
 # Optionally set corporate mirror for apk
diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile
index 1eb56633c561e..3adef53cd0606 100644
--- a/docker/datahub-mce-consumer/Dockerfile
+++ b/docker/datahub-mce-consumer/Dockerfile
@@ -6,12 +6,12 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
 ARG GITHUB_REPO_URL=https://github.com
 ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
 
-FROM golang:1-alpine3.20 AS binary
+FROM golang:1-alpine3.21 AS binary
 
 # Re-declaring arg from above to make it available in this stage (will inherit default value)
 ARG ALPINE_REPO_URL
 
-ENV DOCKERIZE_VERSION=v0.6.1
+ENV DOCKERIZE_VERSION=v0.9.1
 WORKDIR /go/src/github.com/jwilder
 
 # Optionally set corporate mirror for apk
diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile
index 3d59a903414b1..a8ef4e8034fdd 100644
--- a/docker/datahub-upgrade/Dockerfile
+++ b/docker/datahub-upgrade/Dockerfile
@@ -6,12 +6,12 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
 ARG GITHUB_REPO_URL=https://github.com
 ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
 
-FROM golang:1-alpine3.20 AS binary
+FROM golang:1-alpine3.21 AS binary
 
 # Re-declaring arg from above to make it available in this stage (will inherit default value)
 ARG ALPINE_REPO_URL
 
-ENV DOCKERIZE_VERSION=v0.6.1
+ENV DOCKERIZE_VERSION=v0.9.1
 WORKDIR /go/src/github.com/jwilder
 
 # Optionally set corporate mirror for apk
diff --git a/docker/elasticsearch-setup/Dockerfile b/docker/elasticsearch-setup/Dockerfile
index 4e64dcbc1e452..1a6fe5bee6c84 100644
--- a/docker/elasticsearch-setup/Dockerfile
+++ b/docker/elasticsearch-setup/Dockerfile
@@ -6,11 +6,11 @@ ARG APP_ENV=prod
 # Defining custom repo urls for use in enterprise environments. Re-used between stages below.
 ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
 
-FROM golang:1-alpine3.20 AS binary
+FROM golang:1-alpine3.21 AS binary
 
 ARG ALPINE_REPO_URL
 
-ENV DOCKERIZE_VERSION=v0.6.1
+ENV DOCKERIZE_VERSION=v0.9.1
 WORKDIR /go/src/github.com/jwilder
 
 # Optionally set corporate mirror for apk
diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile
index dd88060cd7165..324357b942e8e 100644
--- a/docker/kafka-setup/Dockerfile
+++ b/docker/kafka-setup/Dockerfile
@@ -1,4 +1,4 @@
-ARG KAFKA_DOCKER_VERSION=7.4.6
+ARG KAFKA_DOCKER_VERSION=7.7.1
 
 # Defining custom repo urls for use in enterprise environments. Re-used between stages below.
 ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
@@ -22,7 +22,7 @@ ARG ALPINE_REPO_URL
 ARG APACHE_DOWNLOAD_URL
 ARG GITHUB_REPO_URL
 
-ENV KAFKA_VERSION=3.7.1
+ENV KAFKA_VERSION=3.7.2
 ENV SCALA_VERSION=2.13
 
 LABEL name="kafka" version=${KAFKA_VERSION}
diff --git a/docker/mysql-setup/Dockerfile b/docker/mysql-setup/Dockerfile
index b0ca45ad8f6f2..8a2d42bc23318 100644
--- a/docker/mysql-setup/Dockerfile
+++ b/docker/mysql-setup/Dockerfile
@@ -1,11 +1,11 @@
 # Defining custom repo urls for use in enterprise environments. Re-used between stages below.
 ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
 
-FROM golang:1-alpine3.20 AS binary
+FROM golang:1-alpine3.21 AS binary
 
 ARG ALPINE_REPO_URL
 
-ENV DOCKERIZE_VERSION=v0.6.1
+ENV DOCKERIZE_VERSION=v0.9.1
 WORKDIR /go/src/github.com/jwilder
 
 # Optionally set corporate mirror for apk
diff --git a/docker/postgres-setup/Dockerfile b/docker/postgres-setup/Dockerfile
index e145456e807d4..31e9687cea15e 100644
--- a/docker/postgres-setup/Dockerfile
+++ b/docker/postgres-setup/Dockerfile
@@ -1,11 +1,11 @@
 # Defining custom repo urls for use in enterprise environments. Re-used between stages below.
 ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
 
-FROM golang:1-alpine3.20 AS binary
+FROM golang:1-alpine3.21 AS binary
 
 ARG ALPINE_REPO_URL
 
-ENV DOCKERIZE_VERSION=v0.6.1
+ENV DOCKERIZE_VERSION=v0.9.1
 WORKDIR /go/src/github.com/jwilder
 
 # Optionally set corporate mirror for apk
diff --git a/docker/profiles/docker-compose.actions.yml b/docker/profiles/docker-compose.actions.yml
index c2985f4299326..459fffdd8acf3 100644
--- a/docker/profiles/docker-compose.actions.yml
+++ b/docker/profiles/docker-compose.actions.yml
@@ -6,7 +6,7 @@ x-search-datastore-elasticsearch-env: &search-datastore-env
 
 x-datahub-actions-service: &datahub-actions-service
   hostname: actions
-  image: ${DATAHUB_ACTIONS_IMAGE:-${DATAHUB_ACTIONS_REPO:-acryldata}/datahub-actions}:${ACTIONS_VERSION:-v0.1.1}
+  image: ${DATAHUB_ACTIONS_IMAGE:-${DATAHUB_ACTIONS_REPO:-acryldata}/datahub-actions}:${ACTIONS_VERSION:-v0.1.6}
   env_file:
     - datahub-actions/env/docker.env
     - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env}
diff --git a/docker/profiles/docker-compose.frontend.yml b/docker/profiles/docker-compose.frontend.yml
index b5b2d50143927..c6b15a7016670 100644
--- a/docker/profiles/docker-compose.frontend.yml
+++ b/docker/profiles/docker-compose.frontend.yml
@@ -16,7 +16,7 @@ x-datahub-frontend-service: &datahub-frontend-service
 
 x-datahub-frontend-service-dev: &datahub-frontend-service-dev
   <<: *datahub-frontend-service
-  image: ${DATAHUB_FRONTEND_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-frontend-react}:debug
+  image: ${DATAHUB_FRONTEND_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-frontend-react}:${DATAHUB_VERSION:-debug}
   ports:
     - ${DATAHUB_MAPPED_FRONTEND_DEBUG_PORT:-5002}:5002
     - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml
index 6e3e5780506ac..64163ef970080 100644
--- a/docker/profiles/docker-compose.gms.yml
+++ b/docker/profiles/docker-compose.gms.yml
@@ -40,7 +40,6 @@ x-kafka-env: &kafka-env
   # KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
   SCHEMA_REGISTRY_TYPE: INTERNAL
   KAFKA_SCHEMAREGISTRY_URL: http://datahub-gms:8080/schema-registry/api/
-  SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET: ${SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET:-earliest}
 
 x-datahub-quickstart-telemetry-env: &datahub-quickstart-telemetry-env
   DATAHUB_SERVER_TYPE: ${DATAHUB_SERVER_TYPE:-quickstart}
@@ -73,7 +72,7 @@ x-datahub-system-update-service: &datahub-system-update-service
 
 x-datahub-system-update-service-dev: &datahub-system-update-service-dev
   <<: *datahub-system-update-service
-  image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:debug
+  image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:${DATAHUB_VERSION:-debug}
   ports:
     - ${DATAHUB_MAPPED_UPGRADE_DEBUG_PORT:-5003}:5003
   environment: &datahub-system-update-dev-env
@@ -100,7 +99,8 @@ x-datahub-gms-service: &datahub-gms-service
     - ${DATAHUB_LOCAL_GMS_ENV:-empty2.env}
   environment: &datahub-gms-env
     <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
-    ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:-search_config.yaml}
+    ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
+    STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true}
   healthcheck:
     test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health
     start_period: 90s
@@ -115,7 +115,7 @@ x-datahub-gms-service: &datahub-gms-service
 
 x-datahub-gms-service-dev: &datahub-gms-service-dev
   <<: *datahub-gms-service
-  image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-gms}:debug
+  image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-gms}:${DATAHUB_VERSION:-debug}
   ports:
     - ${DATAHUB_MAPPED_GMS_DEBUG_PORT:-5001}:5001
     - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
@@ -125,7 +125,6 @@ x-datahub-gms-service-dev: &datahub-gms-service-dev
     - ${DATAHUB_LOCAL_GMS_ENV:-empty2.env}
   environment: &datahub-gms-dev-env
     <<: [*datahub-dev-telemetry-env, *datahub-gms-env]
-    ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:-search_config.yaml}
     SKIP_ELASTICSEARCH_CHECK: false
     JAVA_TOOL_OPTIONS: '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5001'
     BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE: false
@@ -159,7 +158,7 @@ x-datahub-mae-consumer-service: &datahub-mae-consumer-service
 
 x-datahub-mae-consumer-service-dev: &datahub-mae-consumer-service-dev
   <<: *datahub-mae-consumer-service
-  image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:debug
+  image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:${DATAHUB_VERSION:-debug}
   environment:
     <<: [*datahub-dev-telemetry-env, *datahub-mae-consumer-env]
   volumes:
@@ -182,10 +181,12 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service
     - ${DATAHUB_LOCAL_MCE_ENV:-empty2.env}
   environment: &datahub-mce-consumer-env
     <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
+    ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
+    STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true}
 
 x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev
   <<: *datahub-mce-consumer-service
-  image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:debug
+  image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:${DATAHUB_VERSION:-debug}
   environment:
     <<: [*datahub-dev-telemetry-env, *datahub-mce-consumer-env]
   volumes:
diff --git a/docker/profiles/docker-compose.prerequisites.yml b/docker/profiles/docker-compose.prerequisites.yml
index eed23a749628f..c9cb444d57c98 100644
--- a/docker/profiles/docker-compose.prerequisites.yml
+++ b/docker/profiles/docker-compose.prerequisites.yml
@@ -135,7 +135,7 @@ services:
   mysql-setup-dev:
     <<: *mysql-setup
     profiles: *mysql-profiles-dev
-    image: ${DATAHUB_MYSQL_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mysql-setup}:debug
+    image: ${DATAHUB_MYSQL_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mysql-setup}:${DATAHUB_VERSION:-debug}
   postgres:
     profiles: *postgres-profiles
     hostname: postgres
@@ -166,7 +166,7 @@ services:
   postgres-setup-dev:
     <<: *postgres-setup
     profiles: *postgres-profiles-dev
-    image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-postgres-setup}:debug
+    image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-postgres-setup}:${DATAHUB_VERSION:-debug}
   cassandra:
     profiles: *cassandra-profiles
     hostname: cassandra
@@ -272,7 +272,7 @@ services:
     environment:
       <<: *kafka-setup-env
       DATAHUB_PRECREATE_TOPICS: ${DATAHUB_PRECREATE_TOPICS:-true}
-    image: ${DATAHUB_KAFKA_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-kafka-setup}:debug
+    image: ${DATAHUB_KAFKA_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-kafka-setup}:${DATAHUB_VERSION:-debug}
   elasticsearch:
     profiles: *elasticsearch-profiles
     hostname: search
@@ -296,7 +296,7 @@ services:
     volumes:
       - esdata:/usr/share/elasticsearch/data
   elasticsearch-setup-dev: &elasticsearch-setup-dev
-    image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-elasticsearch-setup}:debug
+    image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-debug}
     profiles: *elasticsearch-profiles
     hostname: elasticsearch-setup
     env_file: elasticsearch-setup/env/docker.env
@@ -347,7 +347,7 @@ services:
     <<: *opensearch-setup
     profiles: *opensearch-profiles-dev
     hostname: opensearch-setup-dev
-    image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-elasticsearch-setup}:debug
+    image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-debug}
     environment:
       <<: *search-datastore-environment
       USE_AWS_ELASTICSEARCH: ${USE_AWS_ELASTICSEARCH:-true}
diff --git a/docs-website/adoptionStoriesIndexes.json b/docs-website/adoptionStoriesIndexes.json
index f7b78e6cab8b2..618cd4bbb3334 100644
--- a/docs-website/adoptionStoriesIndexes.json
+++ b/docs-website/adoptionStoriesIndexes.json
@@ -20,6 +20,16 @@
       "category": "Financial & Fintech",
       "description": "<i>\"We found DataHub to provide excellent coverage for our needs. What we appreciate most about DataHub is <b>its powerful API platform.</b>\"<br /><br /><div style='color: gray;'>— Jean-Pierre Dijcks, Sr. Dir. Product Management at VISA</div></i><br />"
     },
+    {
+      "name": "Apple",
+      "slug": "apple",
+      "link": "https://youtu.be/5eFZuzG4c-s?feature=shared",
+      "linkType": "video",
+      "tagline": "How Apple built a solid foundation for observability, governance, and data sharing with DataHub",
+      "category": "B2B & B2C",
+      "platform": "cloud",
+      "description": "Discover how DataHub provides a solid foundation for observability, governance, and data sharing, while we explore its role in managing AI and data metadata."
+    },
     {
       "name": "Notion",
       "slug": "notion",
diff --git a/docs-website/build.gradle b/docs-website/build.gradle
index 1860b4a49ae23..1be790695e87e 100644
--- a/docs-website/build.gradle
+++ b/docs-website/build.gradle
@@ -14,7 +14,7 @@ node {
   }
 
   // Version of node to use.
-  version = '21.2.0'
+  version = '22.12.0'
 
   // Version of Yarn to use.
   yarnVersion = '1.22.22'
@@ -83,11 +83,7 @@ task yarnInstall(type: YarnTask) {
 task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall,
     generateGraphQLSchema, generateJsonSchema,
     ':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen',
-    ':metadata-ingestion:buildWheel',
-    ':metadata-ingestion-modules:airflow-plugin:buildWheel',
-    ':metadata-ingestion-modules:dagster-plugin:buildWheel',
-    ':metadata-ingestion-modules:prefect-plugin:buildWheel',
-    ':metadata-ingestion-modules:gx-plugin:buildWheel',
+    ':python-build:buildWheels',
     ]) {
   inputs.files(projectMdFiles)
   outputs.cacheIf { true }
diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
index b0cf5bfa35eca..31644f459ed73 100644
--- a/docs-website/docusaurus.config.js
+++ b/docs-website/docusaurus.config.js
@@ -65,6 +65,14 @@ module.exports = {
     //     isCloseable: false,
     //   },
     // }),
+    announcementBar: {
+          id: "announcement-3",
+          content:
+            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><!--div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div--><p>Watch Metadata & AI Summit sessions on-demand.</p><a href="https://www.youtube.com/@DataHubProject/videos" target="_blank" class="button">Watch Now<span> →</span></a></div>',
+          backgroundColor: "#111",
+          textColor: "#ffffff",
+          isCloseable: false,
+        },
     colorMode: {
       // Only support light mode.
       defaultMode: 'light',
@@ -80,7 +88,26 @@ module.exports = {
       },
       items: [
         {
-          to: "cloud/",
+          type: "dropdown",
+          label: "Solutions",
+          position: "right",
+          items: [
+            {
+              to: "/solutions/discovery",
+              label: "Discovery",
+            },
+            {
+              to: "/solutions/observability",
+              label: "Observability",
+            },
+            {
+              to: "/solutions/governance",
+              label: "Governance",
+            },
+          ]
+        },
+        {
+          to: "/cloud",
           activeBasePath: "cloud",
           label: "Cloud",
           position: "right",
@@ -172,7 +199,7 @@ module.exports = {
                 opacity: 0.8;
               }
             </style>
-            <div class='cloud-cta button button--primary' alt='try-datahub-cloud' style='font-weight: 700;'>Try DataHub Cloud Free</div>
+            <div class='cloud-cta button button--primary' alt='try-datahub-cloud' style='font-weight: 700;'>Get DataHub Cloud</div>
           `,
           position: "right",
         }
diff --git a/docs-website/filterTagIndexes.json b/docs-website/filterTagIndexes.json
index 2309593b2c3b9..b269f23cccd66 100644
--- a/docs-website/filterTagIndexes.json
+++ b/docs-website/filterTagIndexes.json
@@ -231,6 +231,17 @@
         "Features": "Stateful Ingestion, UI Ingestion, Status Aspect"
       }
     },
+    {
+      "Path": "https://hudi.apache.org/docs/syncing_datahub/",
+      "imgPath": "img/logos/platforms/hudi.png",
+      "Title": "Apache Hudi",
+      "Description": "Apache Hudi is an open-source data lake framework that provides ACID transactions, efficient upserts, time travel queries, and incremental data processing for large-scale datasets.",
+      "tags": {
+        "Platform Type": "Datastore",
+        "Connection Type": "Pull",
+        "Features": ""
+      }
+    },
     {
       "Path": "docs/generated/ingestion/sources/iceberg",
       "imgPath": "img/logos/platforms/iceberg.png",
diff --git a/docs-website/generateDocsDir.ts b/docs-website/generateDocsDir.ts
index 032e912c7190e..ad82a85f9e567 100644
--- a/docs-website/generateDocsDir.ts
+++ b/docs-website/generateDocsDir.ts
@@ -284,6 +284,10 @@ function markdown_add_slug(
 //   );
 // }
 
+function trim_anchor_link(url: string): string {
+  return url.replace(/#.+$/, "");
+}
+
 function new_url(original: string, filepath: string): string {
   if (original.toLowerCase().startsWith(HOSTED_SITE_URL)) {
     // For absolute links to the hosted docs site, we transform them into local ones.
@@ -313,7 +317,7 @@ function new_url(original: string, filepath: string): string {
   }
 
   // Now we assume this is a local reference.
-  const suffix = path.extname(original);
+  const suffix = path.extname(trim_anchor_link(original));
   if (
     suffix == "" ||
     [
@@ -335,7 +339,7 @@ function new_url(original: string, filepath: string): string {
     // A reference to a file or directory in the Github repo.
     const relation = path.dirname(filepath);
     const updated_path = path.normalize(`${relation}/${original}`);
-    const check_path = updated_path.replace(/#.+$/, "");
+    const check_path = trim_anchor_link(updated_path);
     if (
       !fs.existsSync(`../${check_path}`) &&
       actually_in_sidebar(filepath) &&
@@ -569,26 +573,20 @@ function write_markdown_file(
 
 function copy_python_wheels(): void {
   // Copy the built wheel files to the static directory.
-  const wheel_dirs = [
-    "../metadata-ingestion/dist",
-    "../metadata-ingestion-modules/airflow-plugin/dist",
-    "../metadata-ingestion-modules/dagster-plugin/dist",
-    "../metadata-ingestion-modules/prefect-plugin/dist",
-    "../metadata-ingestion-modules/gx-plugin/dist",
-  ];
+  // Everything is copied to the python-build directory first, so
+  // we just need to copy from there.
+  const wheel_dir = "../python-build/wheels";
 
   const wheel_output_directory = path.join(STATIC_DIRECTORY, "wheels");
   fs.mkdirSync(wheel_output_directory, { recursive: true });
 
-  for (const wheel_dir of wheel_dirs) {
-    const wheel_files = fs.readdirSync(wheel_dir);
-    for (const wheel_file of wheel_files) {
-      const src = path.join(wheel_dir, wheel_file);
-      const dest = path.join(wheel_output_directory, wheel_file);
+  const wheel_files = fs.readdirSync(wheel_dir);
+  for (const wheel_file of wheel_files) {
+    const src = path.join(wheel_dir, wheel_file);
+    const dest = path.join(wheel_output_directory, wheel_file);
 
-      // console.log(`Copying artifact ${src} to ${dest}...`);
-      fs.copyFileSync(src, dest);
-    }
+    // console.log(`Copying artifact ${src} to ${dest}...`);
+    fs.copyFileSync(src, dest);
   }
 }
 
diff --git a/docs-website/package.json b/docs-website/package.json
index e1130148864e5..52f04e46c42aa 100644
--- a/docs-website/package.json
+++ b/docs-website/package.json
@@ -35,6 +35,7 @@
     "@octokit/plugin-throttling": "^3.5.1",
     "@octokit/rest": "^18.6.2",
     "@radix-ui/react-visually-hidden": "^1.0.2",
+    "@servicebell/widget": "^0.1.6",
     "@supabase/supabase-js": "^2.33.1",
     "@swc/core": "^1.4.2",
     "antd": "^5.0.7",
@@ -47,6 +48,7 @@
     "markprompt": "^0.1.7",
     "react": "^18.2.0",
     "react-dom": "18.2.0",
+    "react-use-draggable-scroll": "^0.4.7",
     "sass": "^1.43.2",
     "swc-loader": "^0.2.6",
     "swiper": "^11.1.4",
diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 06838f2e686bc..c18d8671318f6 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -101,18 +101,42 @@ module.exports = {
         {
           label: "Automations",
           type: "category",
+          collapsed: false,
           items: [
             {
               label: "Documentation Propagation",
               type: "doc",
               id: "docs/automations/docs-propagation",
             },
+            {
+              label: "Glossary Term Propagation",
+              type: "doc",
+              id: "docs/automations/glossary-term-propagation",
+            },
+            {
+              label: "BigQuery Metadata Sync",
+              type: "doc",
+              id: "docs/automations/bigquery-metadata-sync",
+              className: "saasOnly",
+            },
             {
               label: "Snowflake Tag Sync",
               type: "doc",
               id: "docs/automations/snowflake-tag-propagation",
               className: "saasOnly",
             },
+            {
+              label: "AI Classification",
+              type: "doc",
+              id: "docs/automations/ai-term-suggestion",
+              className: "saasOnly",
+            },
+            {
+              label: "AI Documentation",
+              type: "doc",
+              id: "docs/automations/ai-docs",
+              className: "saasOnly",
+            },
           ],
         },
         {
@@ -125,6 +149,25 @@ module.exports = {
           type: "doc",
           id: "docs/glossary/business-glossary",
         },
+        {
+          label: "Compliance Forms",
+          type: "category",
+          collapsed: true,
+          items: [
+            {
+              type: "doc",
+              id: "docs/features/feature-guides/compliance-forms/overview",
+            },
+            {
+              type: "doc",
+              id: "docs/features/feature-guides/compliance-forms/create-a-form",
+            },
+            {
+              type: "doc",
+              id: "docs/features/feature-guides/compliance-forms/complete-a-form",
+            },
+          ],
+        },
         {
           label: "Data Contract",
           type: "doc",
@@ -140,7 +183,6 @@ module.exports = {
           type: "doc",
           id: "docs/features/dataset-usage-and-query-history",
         },
-        "docs/features/feature-guides/documentation-forms",
         {
           label: "Domains",
           type: "doc",
@@ -197,7 +239,21 @@ module.exports = {
           type: "doc",
           id: "docs/posts",
         },
-        "docs/features/feature-guides/properties",
+        {
+          label: "Properties",
+          type: "category",
+          collapsed: true,
+          items: [
+            {
+              type: "doc",
+              id: "docs/features/feature-guides/properties/overview",
+            },
+            {
+              type: "doc",
+              id: "docs/features/feature-guides/properties/create-a-property",
+            },
+          ],
+        },
         {
           label: "Schema history",
           type: "doc",
@@ -231,35 +287,30 @@ module.exports = {
       items: [
         "docs/managed-datahub/welcome-acryl",
         {
-          type: "doc",
-          id: "docs/managed-datahub/approval-workflows",
-          className: "saasOnly",
-        },
-        {
-          "DataHub API": [
+          "Configure Single Sign-On": [
             {
               type: "doc",
-              id: "docs/managed-datahub/datahub-api/entity-events-api",
+              id: "docs/authentication/guides/sso/initialize-oidc",
               className: "saasOnly",
             },
             {
-              "GraphQL API": [
-                "docs/managed-datahub/datahub-api/graphql-api/getting-started",
-              ],
+              type: "doc",
+              id: "docs/managed-datahub/integrations/oidc-sso-integration",
+              className: "saasOnly",
             },
           ],
         },
         {
-          Integrations: [
+          "DataHub API": [
             {
               type: "doc",
-              id: "docs/managed-datahub/integrations/aws-privatelink",
+              id: "docs/managed-datahub/datahub-api/entity-events-api",
               className: "saasOnly",
             },
             {
-              type: "doc",
-              id: "docs/managed-datahub/integrations/oidc-sso-integration",
-              className: "saasOnly",
+              "GraphQL API": [
+                "docs/managed-datahub/datahub-api/graphql-api/getting-started",
+              ],
             },
           ],
         },
@@ -283,7 +334,7 @@ module.exports = {
           ],
         },
         {
-          "Operator Guide": [
+          "Operator Guides": [
             {
               type: "doc",
               id: "docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor",
@@ -294,8 +345,18 @@ module.exports = {
               id: "docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge",
               className: "saasOnly",
             },
+            {
+              type: "doc",
+              id: "docs/managed-datahub/integrations/aws-privatelink",
+              className: "saasOnly",
+            },
           ],
         },
+        {
+          type: "doc",
+          id: "docs/managed-datahub/approval-workflows",
+          className: "saasOnly",
+        },
         {
           type: "doc",
           id: "docs/managed-datahub/chrome-extension",
@@ -307,6 +368,7 @@ module.exports = {
         },
         {
           "DataHub Cloud Release History": [
+            "docs/managed-datahub/release-notes/v_0_3_7",
             "docs/managed-datahub/release-notes/v_0_3_6",
             "docs/managed-datahub/release-notes/v_0_3_5",
             "docs/managed-datahub/release-notes/v_0_3_4",
@@ -479,6 +541,7 @@ module.exports = {
         },
 
         "docs/platform-instances",
+        "docs/lineage/sql_parsing",
         "metadata-ingestion/docs/dev_guides/stateful",
         "metadata-ingestion/docs/dev_guides/classification",
         "metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source",
@@ -531,6 +594,7 @@ module.exports = {
         {
           "Frontend Authentication": [
             "docs/authentication/guides/jaas",
+            "docs/authentication/guides/sso/initialize-oidc",
             "docs/authentication/guides/sso/configure-oidc-react",
             "docs/authentication/guides/sso/configure-oidc-behind-proxy",
           ],
@@ -958,6 +1022,7 @@ module.exports = {
     // "metadata-ingestion/examples/structured_properties/README"
     // "smoke-test/tests/openapi/README"
     // "docs/SECURITY_STANCE"
+    // "metadata-integration/java/datahub-schematron/README"
     // ],
   ],
 };
diff --git a/docs-website/src/pages/_components/CaseStudy/case-study.module.scss b/docs-website/src/pages/_components/CaseStudy/case-study.module.scss
index e17f1f74656dc..29cade9901c4d 100644
--- a/docs-website/src/pages/_components/CaseStudy/case-study.module.scss
+++ b/docs-website/src/pages/_components/CaseStudy/case-study.module.scss
@@ -42,6 +42,9 @@
       flex: 1;
     }
     .cardLink {
+      user-select: none;
+      -webkit-user-drag: none;
+
       color: #000;
       
       &:hover {
@@ -88,6 +91,8 @@
           z-index: 10;
           filter: brightness(2);
           opacity: .9;
+          user-select: none;
+          -webkit-user-drag: none;
         }
         .cardImageBackground {
           position: absolute;
diff --git a/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js b/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js
index 3c840e4fab6eb..4bb1b769b288a 100644
--- a/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js
+++ b/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js
@@ -9,6 +9,16 @@ const caseStudyData = [
     image: "https://datahubproject.io/img/logos/companies/netflix.png",
     link: "https://datahubproject.io/adoption-stories/#netflix",
   },
+  {
+    title: "A Solid Foundation For Data and AI",
+    description:
+      "How Apple built a solid foundation for observability, governance, and data sharing with DataHub.",
+    tag: "Technology",
+    backgroundImage:
+      "https://upload.wikimedia.org/wikipedia/commons/thumb/5/5a/Aerial_view_of_Apple_Park_dllu.jpg/2560px-Aerial_view_of_Apple_Park_dllu.jpg",
+    image: "/img/logos/companies/apple_text.png",
+    link: "https://datahubproject.io/adoption-stories/#apple",
+  },
   {
     title: "Scaling Data Governance",
     description:
diff --git a/docs-website/src/pages/_components/CaseStudy/index.js b/docs-website/src/pages/_components/CaseStudy/index.js
index 0362dec9a0d3e..27766a8456a6f 100644
--- a/docs-website/src/pages/_components/CaseStudy/index.js
+++ b/docs-website/src/pages/_components/CaseStudy/index.js
@@ -1,11 +1,15 @@
-import React from "react";
+import React, { useRef } from "react";
 import styles from "./case-study.module.scss";
 import Link from '@docusaurus/Link'
-import { Carousel } from "antd";
+import { useDraggable } from "react-use-draggable-scroll";
 import caseStudyData from "./caseStudyContent";
 
 
 const CaseStudy = () => {
+  const containerRef = useRef(null);
+
+  const { events } = useDraggable(containerRef);
+
   return (
     <div className={styles.container}>
       {/* Section-1 */}
@@ -15,16 +19,16 @@ const CaseStudy = () => {
           <p>Across finance, healthcare, e-commerce, and countless more.</p>
         </div>
 
-        <div className={styles.card_row}>
-          <div className={styles.card_row_wrapper} >
+        <div className={styles.card_row} {...events} ref={containerRef}>
+          <div className={styles.card_row_wrapper}>
             {caseStudyData.map((caseStudy) => (
               <div className={styles.card} key={caseStudy.link}>
-                <a className={styles.cardLink} href={caseStudy.link} style={caseStudy.backgroundImage ? null : {
+                <a draggable={false} className={styles.cardLink} href={caseStudy.link} style={caseStudy.backgroundImage ? null : {
                   opacity: .5
                 }}>
                   {caseStudy.tag ? <span className={styles.card_tag}>{caseStudy.tag}</span> : null}
                   <div className={styles.card_image} style={{ backgroundColor: caseStudy.backgroundImage ? null : '#eee' }}>
-                    <img src={caseStudy.image} alt="" />
+                    <img src={caseStudy.image} draggable={false} alt="" />
                     <div className={styles.cardImageBackground} style={{ backgroundImage: `url(${caseStudy.backgroundImage})` }} />
                   </div>
                   <div className={styles.card_heading_div}>
diff --git a/docs-website/src/pages/_components/Community/community.module.scss b/docs-website/src/pages/_components/Community/community.module.scss
index 62f0dc13b110a..65da50435597d 100644
--- a/docs-website/src/pages/_components/Community/community.module.scss
+++ b/docs-website/src/pages/_components/Community/community.module.scss
@@ -221,14 +221,14 @@
 
 .numberContainer {
   display: inline-block;
-  width: 11rem;
+  width: 12rem;
   text-align: right;
 }
 
 .numberChange {
   display: inline-block;
   animation: slideIn 0.5s ease-in-out;
-  width: 11rem;
+  width: 12rem;
 }
 
 
diff --git a/docs-website/src/pages/_components/Community/index.js b/docs-website/src/pages/_components/Community/index.js
index a4f2b2304e51e..20917332c443d 100644
--- a/docs-website/src/pages/_components/Community/index.js
+++ b/docs-website/src/pages/_components/Community/index.js
@@ -2,7 +2,7 @@ import React, { useState, useRef, useEffect } from "react";
 import styles from "./community.module.scss";
 import useBaseUrl from "@docusaurus/useBaseUrl";
 
-const TARGET_COUNT = 11535;
+const TARGET_COUNT = 12219;
 const INCREMENT = 1;
 
 const Community = () => {
diff --git a/docs-website/src/pages/_components/Hero/hero.module.scss b/docs-website/src/pages/_components/Hero/hero.module.scss
index c7f4ec6a0b78f..1850757bd454c 100644
--- a/docs-website/src/pages/_components/Hero/hero.module.scss
+++ b/docs-website/src/pages/_components/Hero/hero.module.scss
@@ -233,6 +233,9 @@
         }
         .hero__cta {
           margin-top: 12px;
+          .cta__primary {
+            margin-bottom: 4px;
+          }
         }
         .hero__footer_cta {
           margin-top: 12px;
diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js
index ca94203e94c06..a61b9d8a402bd 100644
--- a/docs-website/src/pages/_components/Hero/index.js
+++ b/docs-website/src/pages/_components/Hero/index.js
@@ -69,7 +69,7 @@ const Hero = ({ onOpenTourModal }) => {
             </div>
             <div className="hero__cta">
               <Link className="cta__primary" to="/cloud">
-                Book a Demo
+                Get Cloud
               </Link>
               <a
                 className="cta__secondary"
@@ -80,7 +80,7 @@ const Hero = ({ onOpenTourModal }) => {
               </a>
             </div>
             <Link className="hero__footer_cta" to="/docs">
-              Get started with Core →
+              Get started with Open Source →
             </Link>
           </div>
           <div className="hero__img">
diff --git a/docs-website/src/pages/_components/QuickstartContent/index.js b/docs-website/src/pages/_components/QuickstartContent/index.js
index 302fe7f5e8382..c92f9676a0b3c 100644
--- a/docs-website/src/pages/_components/QuickstartContent/index.js
+++ b/docs-website/src/pages/_components/QuickstartContent/index.js
@@ -47,7 +47,7 @@ const QuickstartContent = ({}) => {
       >
         <div className="quickstart__title">The only platform you need.</div>
         <div className="quickstart__subtitle">
-          Unified Discovery, Observability, and Governance for Data and AI.
+          Unified discovery, observability, and governance for data and AI.
         </div>
       </motion.div>
       <div
@@ -87,7 +87,7 @@ const QuickstartContent = ({}) => {
               <div className="quickstart__text__head">{data.title}</div>
               <div className="quickstart__text__desc">
                 <p dangerouslySetInnerHTML={{ __html: data.description }} />
-                {/* <span className="learn_more">Learn More →</span> */}
+                <a className="learn_more" href={data.link}>Learn More →</a>
               </div>
             </div>
             <div className="quickstart__img">
diff --git a/docs-website/src/pages/_components/QuickstartContent/quickstartContent.js b/docs-website/src/pages/_components/QuickstartContent/quickstartContent.js
index f5afc09871977..23e23f1acfc99 100644
--- a/docs-website/src/pages/_components/QuickstartContent/quickstartContent.js
+++ b/docs-website/src/pages/_components/QuickstartContent/quickstartContent.js
@@ -1,25 +1,30 @@
+
 const quickstartData = [
-    {
-      heading: "Discovery",
-      title: "Make data democratization a reality",
-      description:
-        "Enable everyone in your organization to effortlessly discover trustworthy data, with experiences tailored for each persona.<span></span>Eliminate breaking changes with detailed cross-platform and column-level lineage. <span></span> Build confidence in your data with a comprehensive view of business, operational, and technical context, all in one place.",
-      image: "/img/quickstart_discovery.png",
-    },
-    {
-      heading: "Observability",
-      title: "Build trust in your data",
-      description:
-        "Effortlessly detect data quality issues with automated checks and AI-driven anomaly detection. <span></span> Notify your team where they work when issues arise and keep stakeholders in the loop with centralized incident tracking. <span></span> Spend minutes, not days, resolving issues with detailed lineage, documentation, and ownership information all in one place.",
-      image: "/img/quickstart_observability.png",
-    },
-    {
-      heading: "Governance",
-      title: "Minimize compliance risk, effortlessly",
-      description:
-        "Ensure every data asset is accounted for and responsibility governed by defining and enforcing documentation standards. <span></span> Automate your governance program to automatically classify assets as they evolve over time. <span></span> Minimize redundant, manual work with GenAI documentation, AI-driven classification, smart propagation, and more.",
-      image: "/img/quickstart_governance.png",
-    },
-  ];
-  
-  export default quickstartData;
\ No newline at end of file
+  {
+    heading: "Discovery",
+    title: "Make data democratization a reality",
+    description:
+      "Enable everyone in your organization to effortlessly discover trustworthy data, with experiences tailored for each persona.<span></span>Eliminate breaking changes with detailed cross-platform and column-level lineage. <span></span> Build confidence in your data with a comprehensive view of business, operational, and technical context, all in one place.",
+    image: "/img/quickstart_discovery.png",
+    link: "/solutions/discovery",
+  },
+  {
+    heading: "Observability",
+    title: "Build trust in your data",
+    description:
+      "Effortlessly detect data quality issues with automated checks and AI-driven anomaly detection. <span></span> Notify your team where they work when issues arise and keep stakeholders in the loop with centralized incident tracking. <span></span> Spend minutes, not days, resolving issues with detailed lineage, documentation, and ownership information all in one place.",
+    image: "/img/quickstart_observability.png",
+    link: "/solutions/observability",
+  },
+  {
+    heading: "Governance",
+    title: "Minimize compliance risk, effortlessly",
+    description:
+      "Ensure every data asset is accounted for and responsibility governed by defining and enforcing documentation standards. <span></span> Automate your governance program to automatically classify assets as they evolve over time. <span></span> Minimize redundant, manual work with GenAI documentation, AI-driven classification, smart propagation, and more.",
+    image: "/img/quickstart_governance.png",
+    link: "/solutions/governance",
+
+  },
+];
+
+export default quickstartData;
\ No newline at end of file
diff --git a/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss
index 57a3037efda19..27bdca710b821 100644
--- a/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss
+++ b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss
@@ -129,7 +129,7 @@
 
     .quickstart__content {
       display: flex;
-      margin-bottom: 3rem;
+      margin-bottom: 6rem;
       width: 100%;
 
       .quickstart__text {
@@ -193,6 +193,11 @@
           .learn_more {
             color: #1890FF;
             font-weight: 500;
+
+            &:hover {
+              text-decoration: none !important;
+              opacity: 0.8;
+            }
           }
         }
       }
@@ -259,4 +264,4 @@
       }
     }
   }
-}
+}
\ No newline at end of file
diff --git a/docs-website/src/pages/_components/Trial/index.js b/docs-website/src/pages/_components/Trial/index.js
index 63d4a4a2bf1f5..bd170f3d21e6b 100644
--- a/docs-website/src/pages/_components/Trial/index.js
+++ b/docs-website/src/pages/_components/Trial/index.js
@@ -11,7 +11,7 @@ const Trial = ({onOpenTourModal}) => {
           <div className={styles.left_content}>
             <span className="">Discover. Observe. Govern.</span>
             <p className="">
-              Get a free trial of<br/><b>DataHub</b> cloud today.
+              Get started with<br/><b>DataHub</b> cloud today.
             </p>
             <div className={styles.btn_div}>
               <Link to="/cloud">Book a Demo</Link>
@@ -20,7 +20,7 @@ const Trial = ({onOpenTourModal}) => {
                 onClick={onOpenTourModal}
               >Product Tour</a>
             </div>
-            <Link className={styles.start_arrow} to="/docs">Get started with Core →</Link>
+            <Link className={styles.start_arrow} to="/docs">Get started with Open Source →</Link>
           </div>
         </div>
         <div className={styles.trial_right}>
diff --git a/docs-website/src/pages/champions/index.js b/docs-website/src/pages/champions/index.js
index 79ecf915e174d..6fd938344bbe1 100644
--- a/docs-website/src/pages/champions/index.js
+++ b/docs-website/src/pages/champions/index.js
@@ -10,240 +10,108 @@ const championSections = [
   {
     people: [
       {
-        name: "Siladitya Chakraborty",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/siladitya_chakraborty.jpeg",
-        position: "Data Engineer, Adevinta",
-        bio: (
-          <>
-            <p>
-              Submiitted 6 pull requests including improvements on graphQL and search API.
-            </p>
-          </>
-        ),
-        social: {
-          linkedin: "https://www.linkedin.com/in/aditya-0bab9a84/",
-          github: "https://github.com/siladitya2",
-        },
-        location: "Barcelona, Spain"
-      },
-      {
-        name: "Sergio Gómez Villamor",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/champ-img/imgs/datahub-champions/sergio_gomez_villamor.jpg",
-        position: "Tech Lead, Adevinta",
-        bio: (
-          <>
-            <p>
-            Submitted 26 pull requests and raised 4 issues, also featured in "Humans of DataHub."
-            </p>
-          </>
-        ),
-        social: {
-          linkedin: "https://www.linkedin.com/in/sgomezvillamor/",
-          github: "https://github.com/sgomezvillamor/",
-          },
-        location: "Barcelona,Spain"
-        },
-      {
-        name: "Amanda Ng",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/data-practitioners-guild/amanda-ng.png",
-        position: "Lead Software Engineer, Grab",
+        name: "Patrick Braz",
+        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/data-practitioners-guild/patrick-franco-braz.jpeg",
+        position: "Data Engineering Specialist, Grupo Boticário",
         bio: (
-          <>
-            <p>
-            Submitted 9 pull requests and shared Grab's experience adopting and implementing DataHub during the October 2022 Town Hall.
-            </p>
-          </>
-        ),
-        social: {
-          linkedin: "https://sg.linkedin.com/in/amandang19",
-          github: "https://github.com/ngamanda",
-          web: "https://ngamanda.com/",
-          },
-        location: "Singapore"
-        },
-        {
-          name: "Patrick Braz",
-          image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/data-practitioners-guild/patrick-franco-braz.jpeg",
-          position: "Data Engineering Specialist, Grupo Boticário",
-          bio: (
             <>
               <p>
-              Submitted 16 pull requests and 3 issues and regularly provided guidance to Community Members in Slack channels.
+              Submitted 16 pull requests and 3 issues and regularly provides guidance to Community Members in Slack channels.
               </p>
             </>
-          ),
-          social: {
+        ),
+        social: {
             linkedin: "https://www.linkedin.com/in/patrick-franco-braz/",
             github: "https://github.com/PatrickfBraz",
-            },
-          location: "Rio de Janeiro, Brazil"
-        },
-        {
-          name: "Steve Pham",
-          image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/data-practitioners-guild/cuong-pham.jpeg",
-          bio: (
-            <>
-              <p>
-              Submitted 4 pull requests and reliably provided direction to Community Members across all support channels in Slack.
-              </p>
-            </>
-          ),
-          social: {
-            },
-        },
-        {
-          name: "Piotr Skrydalewicz",
-          image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/data-practitioners-guild/initials/ps_white.jpg",
-          position: "Data Engineer",
-          bio: (
-            <>
-              <p>
-                Contributed 5 commits in 2022 to the main DataHub Project and Helm repositories, including Stateful Ingestion support for Presto-on-Hive.
-              </p>
-            </>
-          ),
-          social: {
-            linkedin: "https://www.linkedin.com/in/skrydal",
-          },
-          location: "Lodz, Poland"
-        },
-      {
-        name: "Harvey Li",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/data-practitioners-guild/initials/hl_black.jpg",
-        position: "Lead Data Engineer, Grab",
-        bio: (
-          <>
-            <p>
-            Shared Grab's experience adopting and implementing DataHub during the October 2022 Town Hall and featured in Humans of DataHub.
-            </p>
-          </>
-        ),        
-        social: {
-          linkedin: "https://www.linkedin.com/in/li-haihui",
-          github: "https://github.com/HarveyLeo",
-          },
-        location: "Singapore"
         },
+        location: "Rio de Janeiro, Brazil"
+      },
       {
-        name: "Fredrik Sannholm",
-        position: "Wolt",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/data-practitioners-guild/initials/fs_black.jpg",
+        name: "Mike Burke",
+        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/mb.jpg",
         position: "",
         bio: (
           <>
             <p>
-            Drove DataHub adoption at Wolt and featured in Humans of DataHub.           
-            </p>
+            Regularly provides support to Community Members and amplifies DataHub across his network.
+           </p>
           </>
-        ),        
+        ),
         social: {
           },
-        location: "Finland"
-        },
-        {
-          name: "Scott Maciej",
-          position: "Optum",
-          image: "https://raw.githubusercontent.com/datahub-project/static-assets/champ-img/imgs/datahub-champions/initials/sm.jpg",
-          bio: (
-            <>
-              <p>
-              Drove DataHub's adaptation and implementation at Optum.
-              </p>
-            </>
-          ),
-          social: {
-            linkedin: "https://www.linkedin.com/in/scottmaciej/",
-            github: "https://github.com/sgm44",
-            web: "https://www.badhabitbeer.com/",
-            },
-          location: "USA"
       },
       {
-        name: "Tim Bossenmaier",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/champ-img/imgs/datahub-champions/tim_bossenmaier.jpeg",
-        position: "Data & Software Engineer, Cloudflight",
+        name: "Siladitya Chakraborty",
+        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/siladitya_chakraborty.jpeg",
+        position: "Data Engineer, Adevinta",
         bio: (
           <>
             <p>
-            Reliably provides direction to community members and submitted 5 pull request, including improvements to Athena ingestion (support for nested schemas) and the REST emitter. 
+              Submitted 16 pull requests including improvements on graphQL and search API.
             </p>
           </>
         ),
         social: {
-          linkedin: "https://www.linkedin.com/in/tim-bossenmaier/",
-          github: "https://github.com/bossenti",
-          },
-        location: "Innsbruck, Austria"
+          linkedin: "https://www.linkedin.com/in/aditya-0bab9a84/",
+          github: "https://github.com/siladitya2",
         },
+        location: "Barcelona, Spain"
+      },
       {
-        name: "Nikola Kasev",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/champ-img/imgs/datahub-champions/nikola_kasev.jpeg",
-        position: "Data Engineer, KPN",
+        name: "Tim Drahn",
+        position: "Optum",
+        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/td.jpg",
+        position: "Solution Architect, Optum Technologies",
         bio: (
           <>
             <p>
-            Reliably provided direction to Community Members across all support channels in Slack.
+            Submitted 2 pull requests and 1 issue while reliably providing direction to Community Members across all support channels in Slack.
             </p>
           </>
         ),
         social: {
-          linkedin: "https://www.linkedin.com/in/nikolakasev",
-          github: "https://github.com/nikolakasev",
+          linkedin: "https://www.linkedin.com/in/tim-drahn-a873532b/",
+          github: "https://github.com/tkdrahn",
           },
-        location: "Haarlem, Noord-holland"
-        },
+        location: "MA, USA"
+      },
       {
-        name: "Nidhin Nandhakumar",
-        position: "Coursera",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/nn.jpg",
+        name: "Steve Pham",
+        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/data-practitioners-guild/cuong-pham.jpeg",
         bio: (
           <>
             <p>
-              Drove DataHub's adaptation and implementation on Coursera.
+            Submitted 4 pull requests and reliably provided direction to Community Members across all support channels in Slack.
             </p>
           </>
         ),
         social: {
           },
-        },
+      },
       {
-        name: "Wu Teng",
-        position: "CashApp",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/wt.jpg",
+        name: "David Schmidt",
+        image: "https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/datahub-champions/david_schmidt.jpeg",
+        position: "Data Engineer, inovex GmbH",
         bio: (
           <>
             <p>
-              Reliably provided direction to Community Members across all support channels in Slack.
-            </p>
+            Regularly provides support to Community Members in Slack, submitted 4 pull requests and 6 issies.
+           </p>
           </>
         ),
         social: {
+          linkedin: "https://www.linkedin.com/in/david-schmidt-de/",
+          github: "https://github.com/DSchmidtDev",
           },
-        location: "Australia"
       },
       {
-        name: "Felipe Gusmao",
-        position: "Zynga",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/fg.jpg",
-        bio: (
-          <>
-            <p>
-            Shared Zynga's experience adopting and implementing DataHub during the September 2023 Town Hall.
-            </p>
-          </>
-        ),
-        social: {
-          },
-        location: "Toronto, Canada"
-       },
-       {
         name: "Sudhakara ST",
         image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/ss.jpg",
         position: "Engineer, Zynga",
         bio: (
           <>
             <p>
-            Reliably provided direction to Community Members across all support channels in Slack and shared Zynga's experience adopting and implementing DataHub during the September 2023 Town Hall.
+            Reliably provides direction to Community Members across all support channels in Slack and shared Zynga's experience adopting and implementing DataHub during the September 2023 Town Hall.
            </p>
           </>
         ),
@@ -253,83 +121,21 @@ const championSections = [
         location: "Bengaluru, India"
       },
       {
-        name: "Bobbie-Jean Nowak",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/bn.jpg",
-        position: "Technical Product Manager,	Optum ",
-        bio: (
-          <>
-            <p>
-            Drove DataHub's adaptation and implementation at Optum.
-            </p>
-          </>
-        ),
-        social: {
-          linkedin: "https://www.linkedin.com/in/bobbie-jean-nowak-a0076b77/",
-          },
-        location: "Minnesota, USA"
-      },
-      {
-        name: "Dima Korendovych",
-        position: "Optum",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/dk.jpg",
-        bio: (
-          <>
-            <p>
-            Drove DataHub's adaptation and implementation at Optum.
-            </p>
-          </>
-        ),
-        social: {
-          },
-        location: "USA"
-      },
-      {
-        name: "Tim Drahn",
-        position: "Optum",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/td.jpg",
-        position: "Solution Architect, Optum Technologies",
-        bio: (
-          <>
-            <p>
-            Submitted 2 pull requests and 1 issue while reliably providing direction to Community Members across all support channels in Slack.
-            </p>
-          </>
-        ),
-        social: {
-          linkedin: "https://www.linkedin.com/in/tim-drahn-a873532b/",
-          github: "https://github.com/tkdrahn",
-          },
-        location: "MA, USA"
-      },
-      {
-        name: "Kate Koy",
-        position: "Optum",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/kk.jpg",
-        bio: (
-          <>
-            <p>
-            Drove DataHub's adaptation and implementation at Optum.
-            </p>
-          </>
-        ),
-        social: {
-          },
-        location: "USA"
-      },
-      {
-        name: "Anjali Arora",
-        position: "Optum",
-        image: "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-champions/initials/aa.jpg",
+        name: "Tim Bossenmaier",
+        image: "https://raw.githubusercontent.com/datahub-project/static-assets/champ-img/imgs/datahub-champions/tim_bossenmaier.jpeg",
+        position: "Data & Software Engineer, Cloudflight",
         bio: (
           <>
             <p>
-            Drove DataHub's adaptation and implementation at Optum.
+            Reliably provides direction to community members and submitted 9 pull request, including improvements to Athena ingestion (support for nested schemas) and the REST emitter. 
             </p>
           </>
         ),
         social: {
+          linkedin: "https://www.linkedin.com/in/tim-bossenmaier/",
+          github: "https://github.com/bossenti",
           },
-          location: "USA"
+        location: "Innsbruck, Austria"
       },
       {
         name: "Raj Tekal",
diff --git a/docs-website/src/pages/cloud/DemoForm/index.jsx b/docs-website/src/pages/cloud/DemoForm/index.jsx
index fcaa1e129d9be..aa9c1b516be56 100644
--- a/docs-website/src/pages/cloud/DemoForm/index.jsx
+++ b/docs-website/src/pages/cloud/DemoForm/index.jsx
@@ -82,7 +82,7 @@ const DemoForm = ({ formId }) => {
         <div className={clsx(styles.formHeader)}>
           <div className={clsx(styles.formTitle)}>Book a Demo</div>
           <div className={clsx(styles.formSubtitle)}>
-            Schedule your personalized demo and get a free trial.
+            Schedule your personalized demo and get a trial.
           </div>
         </div>
         <div id={`hubspotForm-${formId}`} className={styles.hubspotForm}></div> {/* Use unique ID */}
diff --git a/docs-website/src/pages/cloud/DemoForm/styles.module.scss b/docs-website/src/pages/cloud/DemoForm/styles.module.scss
index 4157a228ae739..c56dd1d598605 100644
--- a/docs-website/src/pages/cloud/DemoForm/styles.module.scss
+++ b/docs-website/src/pages/cloud/DemoForm/styles.module.scss
@@ -46,9 +46,7 @@
   .bookButton {
     display: block;
   }
-  .formContainer {
-    display: none;
-  }
+
   .productTourButton {
     text-align: center!important;
   }
diff --git a/docs-website/src/pages/cloud/DemoFormModal/index.jsx b/docs-website/src/pages/cloud/DemoFormModal/index.jsx
new file mode 100644
index 0000000000000..7c890950bda97
--- /dev/null
+++ b/docs-website/src/pages/cloud/DemoFormModal/index.jsx
@@ -0,0 +1,17 @@
+import React, { useEffect } from 'react';
+import styles from "./styles.module.scss";
+import DemoForm from '../DemoForm';
+
+const DemoFormModal = ({ formId, handleCloseModal }) => {
+
+  return (
+    <div className={styles.modal}>
+    <div className={styles.modalContent}>
+      <button className={styles.closeButton} onClick={handleCloseModal}>✕</button>
+      <DemoForm formId={formId} />
+    </div>
+  </div>
+  );
+};
+
+export default DemoFormModal;
diff --git a/docs-website/src/pages/cloud/DemoFormModal/styles.module.scss b/docs-website/src/pages/cloud/DemoFormModal/styles.module.scss
new file mode 100644
index 0000000000000..44461567e578b
--- /dev/null
+++ b/docs-website/src/pages/cloud/DemoFormModal/styles.module.scss
@@ -0,0 +1,45 @@
+
+.modal {
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background: rgba(0, 0, 0, 0.5); /* Modal background */
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  z-index: 1000;
+  overflow-y: scroll;
+
+  .modalContent {
+    width: 90%;
+    max-width: 540px;
+    background: #fff;
+    border-radius: 16px;
+    position: relative;
+    box-shadow: 0px 0px 20px rgba(0, 0, 0, 0.1);
+    animation: modalShow 0.3s ease-in-out;
+
+    .closeButton {
+      position: absolute;
+      top: 1rem;
+      right: 1rem;
+      background: none;
+      border: none;
+      font-size: 1.5rem;
+      cursor: pointer;
+    }
+  }
+}
+
+@keyframes modalShow {
+  from {
+    opacity: 0;
+    transform: translateY(-20px);
+  }
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
diff --git a/docs-website/src/pages/cloud/Enterprise/styles.module.scss b/docs-website/src/pages/cloud/Enterprise/styles.module.scss
index 9d95dcc3990b6..f96ef053fdbf0 100644
--- a/docs-website/src/pages/cloud/Enterprise/styles.module.scss
+++ b/docs-website/src/pages/cloud/Enterprise/styles.module.scss
@@ -17,6 +17,7 @@
 
 .title {
   font-size: 2.8rem;
+  line-height: 3rem;
   font-weight: 600;
   margin-bottom: 2rem;
 }
diff --git a/docs-website/src/pages/cloud/FeatureCards/index.js b/docs-website/src/pages/cloud/FeatureCards/index.js
index 4a45cbcbe1717..52c7a5eec46b4 100644
--- a/docs-website/src/pages/cloud/FeatureCards/index.js
+++ b/docs-website/src/pages/cloud/FeatureCards/index.js
@@ -9,7 +9,7 @@ const data = {
     {
       title: "Data Discovery",
       icon: "/img/assets/data-discovery.svg",
-      cloudPageLink: "https://www.acryldata.io/acryl-datahub",
+      cloudPageLink: "/solutions/discovery",
       cloudBenefits: [
         { text: "Enhanced search ranking", link: "" }, //  →
         { text: "Personalization for every persona", link: "" }, //  →
@@ -27,7 +27,7 @@ const data = {
     {
       title: "Data Observability",
       icon: "/img/assets/data-ob.svg",
-      cloudPageLink: "https://www.acryldata.io/observe",
+      cloudPageLink: "/solutions/observability",
       cloudBenefits: [
         { text: "Continuous data quality monitors", link: "" }, //  →
         { text: "End-to-end data incident tracking & management", link: "" }, //  →
@@ -45,7 +45,7 @@ const data = {
     {
       title: "Data Governance",
       icon: "/img/assets/data-governance.svg",
-      cloudPageLink: "https://www.acryldata.io/acryl-datahub#governance",
+      cloudPageLink: "/solutions/governance",
       cloudBenefits: [
         { text: "Human-assisted asset certification workflows", link: "" }, //  →
         { text: "Automations to enforce governance standards", link: "" }, //  →
diff --git a/docs-website/src/pages/cloud/Hero/index.js b/docs-website/src/pages/cloud/Hero/index.js
index eeb982544ddca..8e4cb2145c514 100644
--- a/docs-website/src/pages/cloud/Hero/index.js
+++ b/docs-website/src/pages/cloud/Hero/index.js
@@ -1,38 +1,56 @@
-import React, { useEffect } from 'react';
+import React, { useState } from 'react';
 import clsx from "clsx";
 import Link from "@docusaurus/Link";
 import styles from "./styles.module.scss";
 import ScrollingCustomers from '../CompanyLogos';
 import DemoForm from '../DemoForm';
+import DemoFormModal from '../DemoFormModal';
 
 const Hero = () => {
+  const [isModalOpen, setIsModalOpen] = useState(false);
+
+  const handleOpenModal = () => setIsModalOpen(true);
+  const handleCloseModal = () => setIsModalOpen(false);
+
   return (
     <header className={clsx("hero", styles.hero)}>
       <div className="container">
         <div className="hero__content">
           <div className="row row__padded">
             <div className={clsx(styles.hero__cta, styles.col, "col col--7")}>
-                <h1 className={clsx("hero__title", styles.hero__title)}>DataHub Cloud</h1>
-                <div className={clsx("hero__subtitle", styles.hero__subtitle)}>
+              <h1 className={clsx("hero__title", styles.hero__title)}>DataHub Cloud</h1>
+              <div className={clsx("hero__subtitle", styles.hero__subtitle)}>
                 Experience the premium version of DataHub
-                  <div style={{ fontWeight: "500" }}>
-                    with Observability and Governance built-in.
-                  </div>
+                <div style={{ fontWeight: "500" }}>
+                  with Observability and Governance built-in.
                 </div>
-                <Link className={clsx(styles.button, styles.bookButton, "button button--primary button--lg")} to="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup">
-                  Book Demo
-                </Link>
-                <Link className={clsx(styles.button, styles.productTourButton, "button button--secondary button--lg")} to="https://www.acryldata.io/tour">
-                  Live Product Tour →
-                </Link>
-                <ScrollingCustomers />
+              </div>
+
+              <button
+                className={clsx(styles.button, styles.bookButton, "button button--primary button--lg")}
+                onClick={handleOpenModal}
+              >
+                Book Demo
+              </button>
+
+              <Link
+                className={clsx(styles.button, styles.productTourButton, "button button--secondary button--lg")}
+                to="https://www.acryldata.io/tour"
+              >
+                Live Product Tour →
+              </Link>
+              <ScrollingCustomers />
             </div>
-            <div className={clsx(styles.col, "col col--5")}>
-               <DemoForm formId="heroForm" />
+            <div className={clsx(styles.col, styles.hideOnMobile, "col col--5")}>
+              <DemoForm formId="heroForm" />
             </div>
           </div>
         </div>
       </div>
+
+      {isModalOpen && (
+        <DemoFormModal formId="heroFormMobile" handleCloseModal={handleCloseModal} />
+      )}
     </header>
   );
 };
diff --git a/docs-website/src/pages/cloud/Hero/styles.module.scss b/docs-website/src/pages/cloud/Hero/styles.module.scss
index 5da028d017155..1f6f36f5cac69 100644
--- a/docs-website/src/pages/cloud/Hero/styles.module.scss
+++ b/docs-website/src/pages/cloud/Hero/styles.module.scss
@@ -64,6 +64,13 @@
   .productTourButton {
     text-align: center!important;
   }
+
+  .hideOnMobile{
+    display: none;
+  }
+  .hero__title, .hero__subtitle {
+    text-align: center!important;
+  }
 }
 
 @media screen  and (min-width: 1000px){
@@ -71,4 +78,4 @@
     padding-left: 0!important;
     margin-left: 0!important;
   }
-}
\ No newline at end of file
+}
diff --git a/docs-website/src/pages/cloud/UnifiedTabs/index.js b/docs-website/src/pages/cloud/UnifiedTabs/index.js
index c0fbc25a8de6b..d17138fcee629 100644
--- a/docs-website/src/pages/cloud/UnifiedTabs/index.js
+++ b/docs-website/src/pages/cloud/UnifiedTabs/index.js
@@ -11,21 +11,21 @@ const TabbedComponent = () => {
       title: 'Discovery',
       description: 'All the search and discovery features of DataHub Core you already love, enhanced.',
       icon: "/img/assets/data-discovery.svg",
-      link: "https://www.acryldata.io/acryl-datahub",
+      link: "https://datahubproject.io/solutions/discovery",
       image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/discovery.webm',
     },
     {
       title: 'Observability',
       description: 'Detect, resolve, and prevent data quality issues before they impact your business. Unify data health signals from all your data quality tools, including dbt tests and more.',
       icon: "/img/assets/data-ob.svg",
-      link: "https://www.acryldata.io/observe",
+      link: "https://datahubproject.io/solutions/observability",
       image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/observe.webm',
     },
     {
       title: 'Governance',
       description: 'Powerful Automation, Reporting and Organizational tools to help you govern effectively.',
       icon: "/img/assets/data-governance.svg",
-      link: "https://www.acryldata.io/acryl-datahub#governance",
+      link: "https://datahubproject.io/solutions/governance",
       image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/governance.webm',
     },
   ];
diff --git a/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss b/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss
index 752bd16471c0b..34d1b8f74fa8e 100644
--- a/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss
+++ b/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss
@@ -10,6 +10,7 @@
 
 .title {
   font-size: 2.5rem;
+  line-height: 2.75rem;
   font-weight: 500;
   margin-bottom: 2rem;
   text-align: center;
diff --git a/docs-website/src/pages/cloud/index.js b/docs-website/src/pages/cloud/index.js
index 99768ca721078..377fe0aa6475e 100644
--- a/docs-website/src/pages/cloud/index.js
+++ b/docs-website/src/pages/cloud/index.js
@@ -1,4 +1,4 @@
-import React from "react";
+import React, { useEffect, useState } from "react";
 import Layout from "@theme/Layout";
 import Link from "@docusaurus/Link";
 import useDocusaurusContext from "@docusaurus/useDocusaurusContext";
@@ -10,15 +10,27 @@ import UnifiedTabs from "./UnifiedTabs";
 import FeatureCards from "./FeatureCards";
 import Hero from "./Hero";
 import DemoForm from "./DemoForm";
+import ServiceBell from "@servicebell/widget";
+import DemoFormModal from "./DemoFormModal";
+
+const SERVICE_BELL_ID = "00892146e5bc46d98d55ecc2b2fa67e2";
 
 function Home() {
   const context = useDocusaurusContext();
   const { siteConfig = {} } = context;
 
+  const [isModalOpen, setIsModalOpen] = useState(false);
+
+  const handleOpenModal = () => setIsModalOpen(true);
+  const handleCloseModal = () => setIsModalOpen(false);
   if (siteConfig.customFields.isSaas) {
     window.location.replace("/docs");
   }
 
+  useEffect(() => {
+    ServiceBell("init", SERVICE_BELL_ID, { hidden: false });
+  }, []);
+
   return !siteConfig.customFields.isSaas ? (
     <Layout
       title={'DataHub Cloud - Unify Data Observability, Governance and Discovery'}
@@ -28,7 +40,7 @@ function Home() {
       <div className={clsx(styles.bgSection)}>
         <UnifiedTabs />
       </div>
-      <FeatureCards/>
+      <FeatureCards />
       <div className={clsx(styles.bgSection)}>
         <Section>
           <Enterprise />
@@ -41,25 +53,33 @@ function Home() {
       <div className={clsx("hero", styles.hero)}>
         <div className="container" style={{ paddingTop: '12vh', paddingBottom: '12vh' }}>
           <div className="row row__padded">
-            <div  className={clsx(styles.col, styles.hero__cta, "col col--7")}>
-              <h1 className={styles.hero__title}>Start your free trial<br/>today.</h1>
-              <div className={clsx(styles.hero__subtitle)}>Unify Discovery, Observability and Governance<br/>for data and AI.</div>
+            <div className={clsx(styles.col, styles.hero__cta, "col col--7")}>
+              <h1 className={styles.hero__title}>Start with DataHub Cloud<br />today.</h1>
+              <div className={clsx(styles.hero__subtitle)}>
+                Unify Discovery, Observability and Governance<br />for data and AI.
+              </div>
               <div>
-                <Link className={clsx(styles.button, styles.bookButton, "button button--primary button--lg")} to="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup">
+                <button
+                  className={clsx(styles.button, styles.bookButton, "button button--primary button--lg")}
+                  onClick={handleOpenModal}
+                >
                   Book Demo
-                </Link>
+                </button>
                 <Link className={clsx(styles.button, styles.productTourButton, "button button--secondary button--lg")} to="https://www.acryldata.io/tour">
                   Live Product Tour →
                 </Link>
               </div>
               <div className="hero__subtitle" />
             </div>
-            <div className={clsx(styles.col, "col col--5")}>
-               <DemoForm formId="footerForm"/>
+            <div className={clsx(styles.col, "col col--5", styles.hideOnMobile)}>
+              <DemoForm formId="footerForm" />
             </div>
           </div>
         </div>
       </div>
+      {isModalOpen && (
+        <DemoFormModal formId="footerFormMobile" handleCloseModal={handleCloseModal} />
+      )}
     </Layout>
   ) : null;
 }
diff --git a/docs-website/src/pages/cloud/styles.module.scss b/docs-website/src/pages/cloud/styles.module.scss
index 0590b8baf12c8..7921cceafce38 100644
--- a/docs-website/src/pages/cloud/styles.module.scss
+++ b/docs-website/src/pages/cloud/styles.module.scss
@@ -24,14 +24,14 @@
   background-color: #1890ff;
   div {
     color: white;
-    font-size: 1rem;
+    font-size: 1.5rem;
     font-weight: 400;
     opacity: .9;
   }
   a {
     cursor: pointer;
     color: white;
-    font-size: 1.2rem;
+    font-size: 1.75rem;
     font-weight: 600;
   }
 }
@@ -86,11 +86,25 @@
 @media screen and (max-width: 999px) {
   .bookButton, .productTourButton {
     display: block;
+    width: 100%;
+    margin-left: 0!important;
   }
 
   .productTourButton {
     text-align: center!important;
   }
+
+
+  .hideOnMobile {
+    display: none;
+  }
+
+  .formContainer {
+    display: none; 
+  }
+  .hero__title, .hero__subtitle {
+    text-align: center!important;
+  }
 }
 
 @media screen  and (min-width: 1000px){
@@ -98,4 +112,4 @@
     padding-left: 0!important;
     margin-left: 0!important;
   }
-}
\ No newline at end of file
+}
diff --git a/docs-website/src/pages/index.js b/docs-website/src/pages/index.js
index 1c36b81a2da95..e1c94780715d3 100644
--- a/docs-website/src/pages/index.js
+++ b/docs-website/src/pages/index.js
@@ -44,7 +44,7 @@ function Home() {
   return !siteConfig.customFields.isSaas ? (
     <Layout
       title={siteConfig.tagline}
-      description="DataHub is a data discovery application built on an extensible data catalog that helps you tame the complexity of diverse data ecosystems."
+      description="DataHub is a metadata management platform, spaning data discovery, observability and governance. It helps you tame the complexity of diverse data ecosystems."
     >
       {isTourModalVisible ? (
         <div className="tourModal">
diff --git a/docs-website/src/pages/solutions/_components/CaseStudy/case-study.module.scss b/docs-website/src/pages/solutions/_components/CaseStudy/case-study.module.scss
new file mode 100644
index 0000000000000..c41a3bde6a52a
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/CaseStudy/case-study.module.scss
@@ -0,0 +1,176 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  font-family: "Manrope";
+}
+.case_study {
+  display: flex;
+  flex-direction: column;
+  width: 100vw;
+  margin: 5rem auto;
+
+  .case_study_heading {
+    color: var(--primitives-text-text-heading, #373A47);
+    text-align: center;
+    font-family: Manrope;
+    font-size: 29.177px;
+    font-style: normal;
+    font-weight: 400;
+    line-height: normal;
+    letter-spacing: 0.292px;
+  }
+
+  .card_row::-webkit-scrollbar {
+    display: none;
+  }
+  .card_row {
+    overflow-x: scroll;
+    width: 100vw;
+    scrollbar-width: none;
+    display: flex;
+    margin-top: 2rem;
+
+    .card_row_wrapper {
+      display: flex;
+      flex-direction: row;
+      align-items: center;
+      margin: auto;
+    }
+    .cardLink {
+      color: #000;
+      
+      &:hover {
+        text-decoration: none;
+      }
+    }
+    
+    .card {
+      max-width: 419px;
+      max-height: 148px;
+      padding: 1.2rem;
+      margin: 1rem 0.5rem;
+      display: flex; 
+      flex-direction: column;
+      align-items: center; 
+      border-radius: var(--number-scales-2s-32, 32px);
+      border: 1px solid var(--semantics-bg-bg-white, #99999930);
+      background: var(--primitives-grays-1, #9999991a);
+
+      transition: all .3s ease-in-out;
+      &:hover {
+        box-shadow: 0px 1px 4px 1px #0000001C;
+        border-color: #33333340;
+        background: var(--primitives-grays-1, #dddddd1a);
+      }
+    
+      .card_image {
+        border-radius: 16px;
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        padding: 0;
+        margin: auto;
+        margin-right: 16px; 
+    
+      }
+    
+      .card_heading_div {
+        width: 70%; 
+        padding: 12px;
+    
+        .card_heading {
+          color: var(--primitives-grays-9, #484C5C);
+          font-family: Manrope;
+          font-size: 1.3rem;
+          font-style: normal;
+          font-weight: 400;
+          line-height: normal;
+          -webkit-line-clamp: 3;
+          -webkit-box-orient: vertical;
+          overflow: hidden;
+          display: -webkit-box;
+        
+        }
+      }
+    }
+    
+  }
+
+  a:hover {
+    text-decoration: none !important;
+  }
+
+  .bottom_line {
+    cursor: pointer;
+    text-decoration: none;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    color: #12b0fb;
+    font-size: 1.1rem;
+    font-weight: 500;
+    margin-top: 50px;
+    margin-bottom: 10px;
+
+    span {
+      line-height: 10px;
+      font-size: 1.5rem;
+      margin-left: 10px;
+    }
+  }
+}
+
+@media (max-width: 800px) {
+  .case_study {
+    .case_study_heading {
+      text-align: center;
+      font-family: "Manrope";
+      width: 80%;
+      margin: auto;
+      font-size: 1.5rem;
+      line-height: normal;
+      font-weight: 400;
+    }
+
+    .card_row {
+      margin-top: 16px;
+      display: flex;
+      justify-content: flex-start;
+
+      .card_row_wrapper {
+        padding: 0 0;
+        width: 100vh;
+        align-items: flex-start;
+        justify-content: flex-start;
+      }
+
+      .card {
+        min-width: 240px;
+        padding: 0.8rem;
+        margin: 0.5rem;
+        display: flex;
+
+        .card_image {
+          width: 50px;
+          height: 50px;
+          margin: 16px auto;
+        }
+
+        .card_heading_div {
+          text-align: left;
+          padding: 8px;
+
+          .card_heading {
+            font-size: 1rem;
+            line-height: 1.2rem;
+          }
+        }
+      }
+    }
+
+    .bottom_line {
+      font-size: 1rem;
+      margin-top: 40px;
+    }
+  }
+}
diff --git a/docs-website/src/pages/solutions/_components/CaseStudy/index.js b/docs-website/src/pages/solutions/_components/CaseStudy/index.js
new file mode 100644
index 0000000000000..9ebffa006e570
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/CaseStudy/index.js
@@ -0,0 +1,34 @@
+import React from "react";
+import styles from "./case-study.module.scss";
+import clsx from "clsx";
+import Link from '@docusaurus/Link'
+
+const CaseStudy = ({ caseStudyContent }) => {
+  const { title, backgroundColor, items } = caseStudyContent;
+  return (
+    <div className={styles.container} style={{ backgroundColor: backgroundColor }}>
+      <div className={styles.case_study}>
+        <div className={styles.case_study_heading}>
+          {title}
+        </div>
+
+        <div className={clsx(styles.card_row)}>
+          <div className={styles.card_row_wrapper}>
+            {items.map((caseStudy) => (
+              <Link className={clsx(styles.card, "row")} key={caseStudy.link} to={caseStudy.link}>
+                  <div className={clsx(styles.card_image, "col col--3")}>
+                    <img src={caseStudy.imgSrc} alt={caseStudy.alt} />
+                  </div>
+                  <div className={clsx(styles.card_heading_div, "col col--8")}>
+                    <div className={styles.card_heading}>{caseStudy.title}</div>
+                  </div>
+              </Link>
+            ))}
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default CaseStudy;
diff --git a/docs-website/src/pages/solutions/_components/Hero/hero.module.scss b/docs-website/src/pages/solutions/_components/Hero/hero.module.scss
new file mode 100644
index 0000000000000..ef2d90404ed08
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Hero/hero.module.scss
@@ -0,0 +1,181 @@
+.hero__container {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  gap: 16px;
+  background-color: #F9F9FA;
+  background-image: url(/img/solutions/hero-background.png);
+  background-position: bottom 0rem center;
+  background-size: contain;
+  background-repeat: no-repeat;
+  max-height: 890px;
+  min-height: 85vh;
+  border-radius: 48px;
+  margin: 3rem 3rem 12rem 3rem;
+  padding: 5rem;
+
+  .hero__topQuote {
+    color: var(--primitives-text-tex-subtext, #777E99);
+    font-family: Manrope;
+    font-size: 1.2rem;
+    font-style: normal;
+    font-weight: 500;
+    line-height: normal;
+    letter-spacing: 0.2rem;
+    text-transform: uppercase;
+    margin-top: 5vh;
+  }
+
+  .hero__title {
+    color: #373A47;
+    font-family: Manrope;
+    font-size: 4rem;
+    font-style: normal;
+    font-weight: 300;
+    line-height: 4.25rem;
+    max-width: 720px;
+    text-align: center;
+  }
+
+  .hero__description {
+    color: var(--primitives-text-text-heading, #373A47);
+    text-align: center;
+    font-family: Manrope;
+    font-size: 1.25rem;
+    font-style: normal;
+    font-weight: 400;
+    line-height: 2.25rem;
+    max-width: 800px;
+    margin-top: 1rem;
+  }
+
+  .cta__tertiary {
+    color: black;
+    font-size: 1rem;
+    margin-top: .5rem;
+    font-weight: 500;
+  }
+
+  .cta__tertiary:hover {
+    color: black;
+    text-decoration: none;
+    opacity: .8;
+  }
+
+  .hero__cta {
+    margin-top: 0rem;
+    display: flex;
+
+    a {
+      cursor: pointer;
+      font-size: 1rem;
+      background-color: white;
+      padding: 4px 20px;
+      border-radius: 50px;
+      margin: 0 10px 0 0;
+      font-weight: 600;
+      text-decoration: none;
+      transition: background-color .2s ease-in-out;
+      &:hover {
+        opacity : 0.8;
+      }
+    }
+
+    .cta__primary {
+      padding: var(--number-scales-2s-12, 8px) var(--number-scales-2s-24, 24px);
+      justify-content: center;
+      align-items: center;
+      gap: var(--number-scales-2s-8, 8px);
+      border-radius: var(--number-scales-2s-full, 999px);
+      background: var(--semantics-surface-default, #1890FF);
+      color: white;
+    }
+
+    .cta__secondary {
+      padding: var(--number-scales-2s-12, 8px) var(--number-scales-2s-24, 24px);
+      justify-content: center;
+      align-items: center;
+      gap: var(--number-scales-2s-8, 8px);
+      border-radius: var(--number-scales-2s-full, 999px);
+      border: 1px solid var(--semantics-border-default, #1890FF);
+      background: var(--semantics-bg-bg-white, #FFF);
+    }
+  }
+
+  .hero__img_container {
+    position: relative;
+
+    .hero__img {
+      z-index: 10;
+      height: 40vh;
+      min-height: 320px;
+      margin-bottom: -8rem;
+    }
+
+    .hero__img_gradient {
+      position: absolute;
+      right: 0;
+      left: 0;
+      bottom: 0;
+      margin-bottom: -8rem;
+      height: 60%;
+      width: min-content;
+      background: linear-gradient(to top, #FFFFFF 15.52%, transparent);
+    }
+  }
+
+  // Mobile adjustments
+  @media (max-width: 768px) {
+    max-height: auto;
+    padding: 1.25rem;
+    margin: 1rem .5rem 6rem .5rem;
+    border-radius: 24px;
+
+    .hero__topQuote {
+      font-size: 1rem;
+      margin-top: 3vh;
+      text-align: center;
+    }
+
+    .hero__title {
+      font-size: 3rem;
+      line-height: 3.5rem;
+      max-width: 100%;
+    }
+
+    .hero__description {
+      font-size: 1rem;
+      line-height: 1.75rem;
+      max-width: 100%;
+    }
+
+    .hero__img_container {
+      .hero__img {
+        height: 30vh;
+        min-height: 200px;
+        margin-bottom: -6rem;
+        object-fit: contain;
+      }
+    }
+
+    .cta__tertiary {
+      font-size: 0.9rem;
+    }
+
+    .hero__cta {
+      a {
+        margin: 0.5rem;
+        padding: 8px 16px;
+        font-size: 0.9rem;
+      }
+
+      .cta__primary {
+        background: #1890FF;
+      }
+
+      .cta__secondary {
+        border: 1px solid #1890FF;
+      }
+    }
+  }
+}
diff --git a/docs-website/src/pages/solutions/_components/Hero/index.js b/docs-website/src/pages/solutions/_components/Hero/index.js
new file mode 100644
index 0000000000000..fd50f2871b9eb
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Hero/index.js
@@ -0,0 +1,45 @@
+import React from 'react';
+import styles from './hero.module.scss';
+import Link from "@docusaurus/Link";
+
+const Hero = ({ onOpenTourModal, heroContent }) => {
+  const { topQuote, title, description, imgSrc } = heroContent
+  return (
+    <div className={styles.hero}>
+      <div className={styles.hero__container} style={{ backgroundImage: `url(/img/solutions/hero-background.png)` }}>
+        <div className={styles.hero__topQuote}>
+          {topQuote}
+        </div>
+        <div className={styles.hero__title}>
+          {title}
+        </div>
+        <p className={styles.hero__description}>{description}</p>
+        <div className={styles.hero__cta}>
+          <Link className={styles.cta__primary} to="/cloud">
+            Get Cloud
+          </Link>
+          <a
+            className={styles.cta__secondary}
+            onClick={onOpenTourModal}
+          >
+            Product Tour
+          </a>
+        </div>
+        <Link className={styles.cta__tertiary} to="/docs">
+            Start with Open Source →
+        </Link>
+        <div style={{ flexGrow: 1 }} />
+        <div className={styles.hero__img_container}>
+          <img
+            src={imgSrc}
+            alt="DataHub Platform Preview"
+            className={styles.hero__img}
+          />
+          <div className={styles.hero__img_gradient} />
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default Hero;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/_components/Integrations/index.js b/docs-website/src/pages/solutions/_components/Integrations/index.js
new file mode 100644
index 0000000000000..763c2a185c7dd
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Integrations/index.js
@@ -0,0 +1,55 @@
+import React, { useRef, useEffect } from "react";
+import styles from "./integrations.module.scss";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+
+const Integrations = () => {
+  const integrationsPath = 'img/solutions/integrations';
+  const hasAnimatedRef = useRef(false);
+  const counterRef = useRef(null);
+  const handleScroll = () => {
+    if (hasAnimatedRef.current) return;
+    if (!counterRef.current) return;
+
+    const { top } = counterRef.current.getBoundingClientRect();
+    const windowHeight = window.innerHeight;
+
+    if (top <= windowHeight) {
+      hasAnimatedRef.current = true;
+      animateNumber();
+    }
+  };
+
+  useEffect(() => {
+    window.addEventListener('scroll', handleScroll);
+    return () => {
+      window.removeEventListener('scroll', handleScroll);
+    }
+  }, [])
+
+  return (
+    <div className={styles.container}>
+      <div className={styles.section_header}>
+          <span>Integrates with your data stack</span>
+      </div>
+      <div className={styles.community_section}>
+            <div className={styles.carouselContainer}>
+              <div className={styles.slider}>
+                <div className={styles.slide_track}>
+                  {[...Array(3)].map((_, i) => (
+                    <React.Fragment key={i}>
+                      {[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11].map((item, index) => (
+                        <div className={styles.slide} key={index} style={{ backgroundImage: `url(${useBaseUrl(`${integrationsPath}/logo-integration-${item}.png`)})` }}>
+                        </div>
+                      ))}
+                    </React.Fragment>
+                  ))}
+                </div>
+              </div>
+          </div>
+      </div>
+      <a href="/integrations">See all →</a>
+    </div>
+  );
+};
+
+export default Integrations;
diff --git a/docs-website/src/pages/solutions/_components/Integrations/integrations.module.scss b/docs-website/src/pages/solutions/_components/Integrations/integrations.module.scss
new file mode 100644
index 0000000000000..07a6a2548e41a
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Integrations/integrations.module.scss
@@ -0,0 +1,134 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  >a {
+    text-decoration: none;
+    text-align: center;
+    margin-top: 1rem;
+    margin-bottom: 1rem;
+    font-size: 1.25rem;
+  }
+
+  .section_header {
+    color: var(--primitives-text-tex-subtext, #777E99);
+    text-align: center;
+    font-family: "Helvetica Neue";
+    font-size: 1.5rem;
+    font-style: normal;
+    font-weight: 400;
+    line-height: normal;
+    letter-spacing: 0.5px;
+
+    margin-bottom: 2rem;
+    position: relative;
+    padding: 0 2rem;
+    display: block;
+    width: 100%;
+
+    &:before, &:after {
+      content: " ";
+      height: 1px;
+      width: calc((100vw - 48rem)/2);
+      background: #D9DBE4;
+      display: block;
+      position: absolute;
+      top: 50%;
+    }
+  
+    &:before {
+      left: 8rem;
+    }
+  
+    &:after {
+      right: 8rem;
+    }
+  }
+}
+
+
+.carouselContainer {
+  width: 100%;
+  min-height: 100px;
+  min-width: 400px;
+  overflow: hidden;
+}
+@media screen and (max-width: 800px) {
+  .carouselContainer {
+    min-width: auto;
+  }
+}
+
+.slider {
+  height: 100px;
+  margin: 1rem auto;
+  overflow: hidden;
+  position: relative;
+  display: flex;
+  align-items: center;
+  width: 100%;
+
+  &::before,
+  &::after {
+    position: absolute;
+    content: '';
+    width: 90%;
+    height: 100%;
+    z-index: 9;
+  }
+
+}
+
+.slider {
+  position: relative;
+}
+
+.slide_track {
+  display: flex;
+  width: max-content;
+  animation: scroll 30s linear infinite;
+}
+
+.slide {
+  width: 100px;
+  height: 100px;
+  margin-left: 52px;
+  display: flex;
+  justify-content: space-between;
+  overflow: hidden;
+  flex-direction: column;
+  align-items: center;
+  position: relative;
+  background-size: cover;
+  background-repeat: no-repeat;
+}
+
+@keyframes scroll {
+  0% {
+    transform: translateX(0);
+  }
+
+  100% {
+    transform: translateX(-50%);
+  }
+}
+
+
+@keyframes slideIn {
+  0% {
+    opacity: 0;
+    transform: translateY(20px);
+  }
+
+  100% {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
+
+@media only screen and (max-width: 800px) {
+  .slide {
+    width: 80px;
+    height: 80px;
+    margin: auto 1rem;
+  }
+}
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/_components/IntegrationsStatic/index.js b/docs-website/src/pages/solutions/_components/IntegrationsStatic/index.js
new file mode 100644
index 0000000000000..17857f7b0360b
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/IntegrationsStatic/index.js
@@ -0,0 +1,34 @@
+import React, { useRef, useEffect } from "react";
+import styles from "./integrations.module.scss";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+
+const Integrations = () => {
+  const integrationsPath = 'img/solutions/integrations-observe';
+
+
+  return (
+    <div className={styles.container}>
+      <div className={styles.section_header}>
+          <span>Integrates with your data stack</span>
+      </div>
+      <div className={styles.community_section}>
+            <div className={styles.carouselContainer}>
+              <div className={styles.slider}>
+                <div className={styles.slide_track}>
+                  {[...Array(1)].map((_, i) => (
+                    <React.Fragment key={i}>
+                      {[1, 2, 3, 4, 5, 6, 7, 8].map((item, index) => (
+                        <div className={styles.slide} key={index} style={{ backgroundImage: `url(${useBaseUrl(`${integrationsPath}/logo-integration-${item}.png`)})` }}>
+                        </div>
+                      ))}
+                    </React.Fragment>
+                  ))}
+                </div>
+              </div>
+          </div>
+      </div>
+    </div>
+  );
+};
+
+export default Integrations;
diff --git a/docs-website/src/pages/solutions/_components/IntegrationsStatic/integrations.module.scss b/docs-website/src/pages/solutions/_components/IntegrationsStatic/integrations.module.scss
new file mode 100644
index 0000000000000..a945f4ae0598f
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/IntegrationsStatic/integrations.module.scss
@@ -0,0 +1,114 @@
+.container {
+  display: flex;
+  flex-direction: column;
+
+  .section_header {
+    color: var(--primitives-text-tex-subtext, #777E99);
+    text-align: center;
+    font-family: "Helvetica Neue";
+    font-size: 1.5rem;
+    font-style: normal;
+    font-weight: 400;
+    line-height: normal;
+    letter-spacing: 0.5px;
+
+    margin-bottom: 2rem;
+    position: relative;
+    padding: 0 2rem;
+    display: block;
+    width: 100%;
+
+    &:before, &:after {
+      content: " ";
+      height: 1px;
+      width: calc((100vw - 65rem)/2);
+      background: #D9DBE4;
+      display: block;
+      position: absolute;
+      top: 50%;
+    }
+  
+    &:before {
+      left: 20rem;
+    }
+  
+    &:after {
+      right: 20rem;
+    }
+  }
+}
+
+
+.carouselContainer {
+  width: 100%;
+  min-height: 100px;
+  min-width: 400px;
+}
+
+.slider {
+  height: 100px;
+  margin: 1rem auto;
+  overflow: hidden;
+  position: relative;
+  display: flex;
+  align-items: center;
+  width: 100%;
+
+  &::before,
+  &::after {
+    position: absolute;
+    content: '';
+    width: 90%;
+    height: 100%;
+    z-index: 9;
+  }
+
+}
+
+.carouselContainer {
+  overflow: hidden;
+}
+
+.slider {
+  position: relative;
+  display: flex;
+}
+
+.slide_track {
+  display: flex;
+  width: 80%;
+  margin: auto;
+  flex-direction: row;
+  align-items: center;
+  justify-content: space-evenly;
+}
+
+.slide {
+  width: 80px;
+  height: 80px;
+  margin: auto 0;
+  display: flex;
+  justify-content: space-between;
+  overflow: hidden;
+  flex-direction: column;
+  align-items: center;
+  position: relative;
+  background-size: cover;
+  background-repeat: no-repeat;
+}
+
+
+@media only screen and (max-width: 800px) {
+  .slider {
+    max-width: 100vw;
+    min-width: auto;
+  }
+  .slide_track {
+    width: 95%;
+  }
+  .slide {
+    width: 40px;
+    height: 40px;
+    margin: auto 0;
+  }
+}
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/_components/Persona/index.js b/docs-website/src/pages/solutions/_components/Persona/index.js
new file mode 100644
index 0000000000000..d377de28525bd
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Persona/index.js
@@ -0,0 +1,46 @@
+import React from "react";
+import styles from "./styles.module.scss";
+import clsx from "clsx";
+
+const Persona = ({ personaContent }) => {
+  const { title, personas } = personaContent;
+
+  return (
+    <div className={styles.container}>
+      <div className={styles.personas}>
+        <div className={styles.persona_heading}>{title}</div>
+
+        <div className={clsx(styles.persona_row)}>
+          <div className={styles.persona_row_wrapper}>
+            <div className={styles.persona_bg_line}></div>
+            {personas.map((persona, index) => (
+              <div key={index} className={clsx(styles.persona)}>
+                <div className={clsx(styles.persona_img)}>
+                  <img src={persona.imgSrc} alt={persona.alt} />
+                </div>
+                <div className={styles.features}>
+                  <FeatureItem text={persona.feature1} />
+                  <FeatureItem text={persona.feature2} />
+                  <FeatureItem text={persona.feature3} />
+                  <FeatureItem text={persona.feature4} />
+                </div>
+              </div>
+            ))}
+          </div>
+        </div>
+        <div className={styles.persona_row_mobile}>
+          <img src='/img/solutions/personas-mobile.png' alt="Persona" />
+        </div>
+      </div>
+      <div className={styles.card_gradient} />
+    </div>
+  );
+};
+
+const FeatureItem = ({ text }) => (
+  <div className={styles.featureItem}>
+    {text}
+  </div>
+);
+
+export default Persona;
diff --git a/docs-website/src/pages/solutions/_components/Persona/styles.module.scss b/docs-website/src/pages/solutions/_components/Persona/styles.module.scss
new file mode 100644
index 0000000000000..3568cc2b75611
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Persona/styles.module.scss
@@ -0,0 +1,163 @@
+.container {
+  padding: 5rem;
+  padding-bottom: 0;
+  display: flex;
+  position: relative;
+  justify-content: center;
+  align-items: center;
+  background: linear-gradient(30deg, #EBEBEB 15.52%, #FFF 85.84%);
+}
+
+.personas {
+  width: 100%;
+}
+
+.persona_heading {
+  color: var(--primitives-text-tex-subtext, #777E99);
+  text-align: center;
+  font-family: Manrope;
+  font-size: 2rem;
+  font-style: normal;
+  font-weight: 500;
+  line-height: normal;
+  letter-spacing: 0.34px;
+  margin-bottom: 2rem;
+  position: relative;
+  padding: 0 2rem;
+  display: block;
+  width: 100%;
+
+  &:before, &:after {
+    content: " ";
+    height: 2px;
+    width: calc((100vw - 1500px)/2);
+    background: #D9DBE4;
+    display: block;
+    position: absolute;
+    top: 50%;
+  }
+
+  &:before {
+    left: 8rem;
+  }
+
+  &:after {
+    right: 8rem;
+  }
+}
+
+.persona_row {
+  display: flex;
+  justify-content: center;
+  gap: 50px;
+  position: relative;
+  padding-top: 20px; /* Add some padding to give space above the line */
+}
+
+.persona_row_mobile {
+  display: none;
+}
+
+.persona_row_wrapper {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  background: inherit; /* Inherit the container's gradient */
+}
+
+.persona {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  min-width: 365px;
+  margin: 2rem;
+  margin-bottom: 0;
+  height: 100%;
+  background: inherit; /* Allow persona card to have gradient */
+}
+
+.persona_img {
+  margin-bottom: 12px;
+  z-index: 1;
+}
+
+.persona_img img {
+  height: 200px;
+}
+
+.features {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.featureItem {
+  display: flex;
+  padding: 0.625rem;
+  border-radius: 0.375rem;
+  background: white;
+  opacity: 0.8;
+  color: var(--primitives-text-tex-subtext, #777E99);
+  font-family: Manrope;
+  font-size: 1rem;
+  font-style: normal;
+  font-weight: 400;
+  line-height: normal;
+  letter-spacing: 0.01rem;
+  text-transform: capitalize;
+  margin: 0.5rem;
+  justify-content: center;
+  width: max-content;
+}
+
+.card_gradient {
+  position: absolute;
+  right: 0;
+  left: 0;
+  bottom: 0;
+  height: 40%;
+  background: linear-gradient(to top, #EBEBEB 15.52%, transparent);
+}
+
+.persona_bg_line {
+  width: 100%;
+  max-width: 900px;
+  height: 0;
+  border-bottom: 1px dashed #aaa;
+  background-size: contain; 
+  position: absolute;
+  top: 28%; 
+  margin: auto;
+}
+
+
+@media (max-width: 768px) {
+  .container {
+    padding: 4rem 2rem;
+    max-width: 100vw;
+  }
+
+  .persona_heading {
+    font-size: 1.4rem;
+  }
+  .persona_row {
+    display: none !important;
+  }
+  .persona_row_mobile {
+    display: flex !important;
+    flex-direction: column;
+    align-items: center;
+  }
+  
+  .persona_row_wrapper {
+    display: block !important;
+  }
+
+  .persona {
+    margin: 3rem auto;
+  }
+
+  .card_gradient {
+    height: 15%;
+  }
+}
diff --git a/docs-website/src/pages/solutions/_components/QuickstartContent/index.js b/docs-website/src/pages/solutions/_components/QuickstartContent/index.js
new file mode 100644
index 0000000000000..422a7b16cc216
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/QuickstartContent/index.js
@@ -0,0 +1,63 @@
+import React, { useEffect, useRef, useState } from "react";
+import clsx from "clsx";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+import styles from "./quickstartcontent.module.scss";
+import { motion, useScroll, useTransform} from 'framer-motion';
+
+const QuickstartContent = ({ quickstartContent }) => {
+  const scrollableElement = useRef(null)
+  const { scrollYProgress } = useScroll({
+    target: scrollableElement,
+    offset: ["start end", "end end"]
+  })
+  const scaleBar = useTransform(scrollYProgress, [0, 0.2, .9, 1], [0, 0, .8, 1]);
+  const opacityBar = useTransform(scrollYProgress, [0, 0.2, 0.4], [0, 0, 1]);
+
+  return (
+    <div className={clsx("quickstart", styles.quickstart)}>
+      <div
+        className="quickstart__container"
+        id="quickstart__container"
+        ref={scrollableElement}
+      >
+        <motion.div
+          className="quickstart__bar"
+          style={{  scaleY: scaleBar, opacity: opacityBar }}
+        />
+        {quickstartContent.map((data, idx) => (
+          <motion.div key={idx} className="quickstart__content"
+            initial={{
+              opacity: 0,
+              scale: .9,
+              y: 50,
+            }}
+            exit={{
+              opacity: 0,
+              scale: .9,
+              y: -50
+            }}
+            whileInView={{
+              opacity: 1,
+              scale: 1,
+              y: 0,
+              transition: {
+                delay: 0,
+                duration: .75
+              }
+            }}
+            viewport={{ once: true, amount: .4 }}
+          >
+            <div className="quickstart__img">
+              <img src={useBaseUrl(data.image)} />
+            </div>
+            <div className="quickstart__text">
+              <div className="quickstart__text__head">{data.title}</div>
+            </div>
+          </motion.div>
+        ))}
+      </div>
+    </div>
+  );
+};
+
+export default QuickstartContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/_components/QuickstartContent/quickstartcontent.module.scss b/docs-website/src/pages/solutions/_components/QuickstartContent/quickstartcontent.module.scss
new file mode 100644
index 0000000000000..bf9369929112a
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/QuickstartContent/quickstartcontent.module.scss
@@ -0,0 +1,118 @@
+@media only screen and (max-width: 800px) {
+  .quickstart {
+    margin: 6rem auto!important;
+    width: 95vw!important;
+  }
+}
+.quickstart {
+  width: 80vw;
+  margin: 12rem auto;
+  display: flex;
+  flex-direction: column;
+
+  :global {
+
+    .quickstart__bar {
+        width: 4px;
+        height: 100%;
+        position: absolute;
+
+        background: linear-gradient(180deg, #546167 0.71%, #58595f 95.37%, #69cfff 100%);
+        border-radius: 10px;
+        transform-origin: top;
+        // transition: transform 0.3s;
+        // animation: progress 0.3s linear;
+
+        /* Inside auto layout */
+        display: block;
+    }
+
+    .quickstart__container {
+      position: relative;
+    }
+
+    .quickstart__content {
+      display: flex;
+      margin: 3rem;
+      width: 100%;
+
+      .quickstart__text {
+        width: 60%;
+        min-width: 600px;
+        padding-right: 2rem;
+        display: flex;
+        justify-content: center;
+        flex-direction: column;
+        
+        div {
+          padding-left: 1rem;
+        }
+
+        .quickstart__text__head {
+
+          /* H4 | Semibold */
+          font-family: 'Manrope';
+          font-style: normal;
+          font-weight: 400;
+          font-size: 2rem;
+          line-height: 2.5rem;
+
+          color: #171B2B;
+          margin-top: 1rem;
+        }
+
+      }
+
+      .quickstart__img {
+        display: flex;
+        align-items: flex-start;
+        justify-content: center;
+        width: 20%;
+
+        img {
+          width: 2rem;
+          min-width: 100px;
+        }
+      }
+    }
+
+
+    @media only screen and (max-width: 800px) {
+      .quickstart__bar {
+        display: none;
+      }
+
+      .quickstart__container {
+        max-width: 95% !important;
+      }
+
+      .quickstart__content {
+        display: flex;
+        margin: 2rem;
+
+        .quickstart__text {
+          min-width: 0;
+          width: 100%;
+          padding-left: 40px;
+          padding-right: 0;
+
+          div {
+            padding-left: 0;
+          }
+          .quickstart__text__head {
+            font-size: 1.25rem;
+            font-weight: 400;
+            line-height: 1.75rem;
+            margin: 0 !important;
+            padding-right: 1.5rem;
+          }
+        }
+        .quickstart__img {
+          display: flex;
+          min-width: 50px;
+          margin: auto;
+        }
+      }
+    }
+  }
+}
diff --git a/docs-website/src/pages/solutions/_components/SlidingTabs/index.js b/docs-website/src/pages/solutions/_components/SlidingTabs/index.js
new file mode 100644
index 0000000000000..2dca5f63ae765
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/SlidingTabs/index.js
@@ -0,0 +1,69 @@
+import React, { useState } from 'react';
+import styles from './styles.module.scss';
+import clsx from 'clsx';
+
+const TabbedComponent = () => {
+  const [activeTab, setActiveTab] = useState(0);
+
+  const tabs = [
+    {
+      title: 'Deploy with enterprise-grade security',
+      description: 'Acryl Observe deploys and runs in your own VPC, offering pre-configured support for advanced networking features—like AWS PrivateLink, or Private Google Access—to facilitate secure, private connectivity to cloud services. Plus, both Observe and Acryl Cloud are certified to meet rigorous compliance and security standards, like SOC 2.',
+      icon: "/img/solutions/lock.png",
+    },
+    {
+      title: 'Scale from Zero to Infinity',
+      description: 'Acryl Observe is built for any scale. Leveraging the power of Acryl Cloud, Observe can scale to support data warehouses with petabytes of data in tens of thousands of tables—and tens of billions of rows. And because it’s a fully managed SaaS offering, it’s also ideal for small organizations still building out their data ecosystems.',
+      icon: "/img/solutions/rocket-launch.png",
+    },
+    {
+      title: 'Reduce tool clutter and operational burden',
+      description: 'Simplify your stack. Avoid duplication across tools by unifying data discovery, data governance, and data quality into one central tool. Skip spending countless engineering hours maintaining inaccessible, code-first data quality frameworks',
+      icon: "/img/solutions/communities.png",
+    },
+    {
+      title: 'Reduce the risk of vendor lock-in',
+      description: 'Get the benefits of open source in a fully managed, limitlessly scalable SaaS offering. Acryl Observe and Acryl Cloud are built on top of the DataHub Project, proven open-source technology with an active, thriving community of contributors and users. Customers get 100% compatibility with open-source DataHub, plus regular updates and improvements, source code transparency, community-based support, proven security, and protection against vendor lock-in.',
+      icon: "/img/solutions/water-lock.png",
+    }
+  ];
+
+  return (
+    <div className={clsx(styles.tabbedComponent)}>
+      <div className={clsx(styles.container)}>
+        <div className={clsx(styles.leftSection)}>
+          <div className={clsx(styles.title)}>
+            Secure. Scalable.<br/>Simple. <span className={clsx(styles.titleBlue)}> Open.</span>
+          </div>
+          <div className={clsx(styles.tabs)}>
+            {tabs.map((tab, index) => (
+              <React.Fragment key={index}>
+                <div className={clsx(styles.tab, { [styles.activeTab]: activeTab === index })}>
+                  <button 
+                    className={clsx(styles.tabButton, { [styles.active]: activeTab === index })}
+                    onClick={() => setActiveTab(index)}
+                  >
+                    <img className={clsx(styles.icon)} src={tab.icon} alt={`${tab.title} icon`} />
+                    <div className={clsx(styles.tabTitle, { [styles.activeTitle]: activeTab === index })}>
+                      {tab.title}
+                    </div>
+                  </button>
+                  {activeTab === index && (
+                    <div className={clsx(styles.dropdown)}>
+                      {tab.description}
+                    </div>
+                  )}
+                </div>
+              </React.Fragment>
+            ))}
+          </div>
+        </div>
+        <div className={clsx(styles.imageContainer)}>
+          <div className={clsx(styles.tabImage)} style={{ backgroundImage: `url(/img/solutions/sliding-tab-bg.png)`}} />
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default TabbedComponent;
diff --git a/docs-website/src/pages/solutions/_components/SlidingTabs/styles.module.scss b/docs-website/src/pages/solutions/_components/SlidingTabs/styles.module.scss
new file mode 100644
index 0000000000000..982234a2b721e
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/SlidingTabs/styles.module.scss
@@ -0,0 +1,171 @@
+.tabbedComponent {
+  padding-top: 48px;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.leftSection {
+  padding-left: 4rem;
+}
+
+.title {
+  color: #373A47;
+  font-family: Manrope;
+  font-size: 3.6rem;
+  font-style: normal;
+  font-weight: 500;
+  line-height: 4.5rem;
+  letter-spacing: -0.18rem;
+  text-align: left;
+  margin-bottom: 3rem;
+
+  .titleBlue {
+    color: var(--semantics-border-default, #1890FF);
+  }
+}
+
+.container {
+  display: flex;
+  flex-direction: column; // Changed to column for mobile view
+  background: white;
+  overflow: hidden;
+  width: 100vw;
+}
+
+.tabs {
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+  text-align: left !important;
+}
+
+.tab {
+  align-items: center;
+  margin: 0.5rem 0;
+  position: relative;
+  
+  &.activeTab {
+    border-left: 3px solid #4C49E4;
+    .tabTitle {
+      color: black !important;
+    }
+    .icon {
+      filter: brightness(0);
+    }
+
+  }
+
+  .tabButton {
+    padding: 0rem 1rem 1rem 1rem;
+    background: none;
+    border: none;
+    cursor: pointer;
+    display: flex;
+    align-items: center;
+    width: 100%;
+    justify-content: left;
+    text-align: left;
+
+    .tabTitle {
+      color: var(--primitives-text-tex-subtext, #777E99);
+      font-family: Manrope;
+      font-size: 1.45rem;
+      font-style: normal;
+      font-weight: 600;
+      line-height: normal;
+      padding-left: 1rem;
+      transition: all .3s;
+    }
+    .icon {
+      transition: all .3s;
+    }
+    &:hover {
+      .tabTitle {
+        color: black;
+      }
+      .icon {
+        filter: brightness(0);
+      }
+    }
+  }
+
+  .dropdown {
+    background-color: #ffffff;
+    margin-top: 5px;
+    padding: 0rem 1.5rem 0.5rem 1.5rem;
+    color: #777E99;
+    font-family: Manrope;
+    font-size: 1.25rem;
+    font-style: normal;
+    font-weight: 500;
+    line-height: 2rem; /* 160% */
+  }
+}
+
+.imageContainer {
+  justify-content: right;
+  background-color: transparent;
+  margin: 1rem 0; 
+  height: 520px;
+  align-self: center;
+  width: 40%;
+  border-radius: 24px;
+  display: flex;
+  flex-grow: 1;
+}
+
+.tabImage {
+  width: 100%;
+  height: 100%;
+  display: flex;
+  background-size: contain;
+  background-repeat: no-repeat;
+  background-position: right center;
+}
+
+
+
+@media (min-width: 799px) {
+  .container {
+    flex-direction: row; // Change back to row for larger screens
+    padding: 40px 0px;
+  }
+  .tabs {
+    width: 800px;
+  }
+
+  .imageContainer {
+    margin: 1rem 0rem 1rem 1rem;
+  }
+}
+
+@media (max-width: 800px) {
+  .title {
+    font-size: 2.5rem;
+    line-height: 3rem;
+  }
+  .tabButton {
+    .icon {
+      height: 24px;
+      width: 24px;
+    }
+    .tabTitle {
+      font-size: 1rem!important;
+      padding-left: .25rem!important;
+    }
+  }
+  .dropdown {
+    font-size: .9rem!important;
+    line-height: 1.5rem!important;
+  }
+
+  .imageContainer {
+    display: none !important;
+  }
+
+  .leftSection {
+    padding: 2rem;
+  }
+  
+}
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/_components/Testimonials/index.js b/docs-website/src/pages/solutions/_components/Testimonials/index.js
new file mode 100644
index 0000000000000..54aaca0ebb5f8
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Testimonials/index.js
@@ -0,0 +1,29 @@
+import React, { useEffect, useRef, useState } from "react";
+import clsx from "clsx";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+import styles from "./styles.module.scss";
+
+const Testimonials = ({ testimonialsData }) => {
+    const { title, feature1, feature2, feature1Link, feature2Link, imgSrc } = testimonialsData;
+  return (
+    <div className={clsx("testimonials", styles.testimonials)}>
+        <div className="testimonials__content">
+        <div className="testimonials__card">
+            <div className="testimonials__text">
+            <div className="testimonials__quote_title">
+                {title}
+            </div>
+            <span className="testimonials__quote_description">
+                Seamlessly integrated with DataHub Cloud's<br/><a href={useBaseUrl(feature1Link)} className="testimonials__quote_black">{feature1}</a> and <a href={useBaseUrl(feature2Link)} className="testimonials__quote_black">{feature2}</a> solutions.
+            </span>
+            </div>
+            <div className="testimonials__logo">
+              <img src={useBaseUrl(imgSrc)} />
+            </div>
+        </div>
+        </div>
+    </div>
+  );
+};
+
+export default Testimonials;
diff --git a/docs-website/src/pages/solutions/_components/Testimonials/styles.module.scss b/docs-website/src/pages/solutions/_components/Testimonials/styles.module.scss
new file mode 100644
index 0000000000000..ce49d7b4f6fa1
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Testimonials/styles.module.scss
@@ -0,0 +1,99 @@
+.testimonials {
+    background: linear-gradient(90deg, #F8F9F9 0%, #E9EAEC 100%);
+  
+    :global {
+      .testimonials__content {
+        width: 80vw;
+        max-width: 1200px;
+        margin: 0 auto;
+        position: relative;
+        padding: 4.5rem 0;
+  
+        .testimonials__card {
+          margin: 0rem 4rem;
+          display: flex;
+  
+          .testimonials__logo {
+            min-width: 100px;
+            margin: 2rem auto;
+            img {
+              max-width: 100%;
+            }
+          }
+  
+          .testimonials__text {
+            width: 94%;
+            padding-left: 2rem;
+            color: #2e2e38;
+
+            .testimonials__quote_title {
+              font-family: "Manrope";
+              font-style: normal;
+              font-weight: 500;
+              font-size: 3.5rem;
+              line-height: 120%;
+              position: relative;
+              margin: 1.8rem auto;
+            }
+
+            .testimonials__quote_description {
+              font-size: 1.8rem;
+              margin-top: 0.5rem;
+              color: #656c77;
+              font-weight: 300;
+              line-height: normal;
+            }
+
+            .testimonials__quote_black {
+              color: #2E2E38;
+
+              &:hover {
+                text-decoration: none;
+                opacity: 0.8;
+                cursor: pointer;
+              }
+            }
+          }
+        }
+      }
+  
+      @media only screen and (max-width: 800px) {
+        .testimonials__content {
+          width: 100vw;
+          padding: 2rem 0;
+          text-align: center;
+          
+          .testimonials__card {
+            flex-direction: column;
+            margin: 2rem 1rem;
+  
+            .testimonials__logo {
+              max-width: 40px;
+              margin-bottom: 16px;
+            }
+  
+            .testimonials__text {
+              width: 100%;
+              padding-left: 0;
+              max-width: 100%;
+              .testimonials__quote_title {
+                font-size: 2.25rem;
+              }
+              .testimonials__quote_description {
+                font-size: 1.5rem;
+              }
+              .testimonials__quote {
+                font-size: 1.1rem;
+                line-height: 1.75rem;
+              }
+              .testimonials__company {
+                font-size: 1rem;
+              }
+            }
+          }
+        }
+        
+      }
+    }
+  }
+  
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/_components/Tiles/index.js b/docs-website/src/pages/solutions/_components/Tiles/index.js
new file mode 100644
index 0000000000000..3b087e97ae20b
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Tiles/index.js
@@ -0,0 +1,80 @@
+import React from "react";
+import styles from "./tiles.module.scss";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+import clsx from "clsx";
+
+const Tiles = ({ tilesContent }) => {
+  const { title, theme, tileItems } = tilesContent;
+
+  const sectionThemeClass = theme === "dark" ? styles.darkSection : styles.lightSection;
+  const itemThemeClass = theme === "dark" ? styles.darkItem : styles.lightItem;
+  const diagramItemThemeClass = theme === "dark" ? styles.darkDiagramItem : styles.lightDiagramItem;
+
+  return (
+    <div className={clsx(sectionThemeClass)}>
+      <div className={styles.ecosystem_section}>
+        <div className={styles.ecosystem_section_content}>
+          <div className={styles.ecosystem_section_lower_content}>
+            <div className={styles.itemWrappers}>
+              <div className={styles.ecosystem_section_upper_content}>
+                <div className={styles.ecosystem_section_heading} dangerouslySetInnerHTML={{ __html: title }}></div>
+              </div>
+              {tileItems.map((item, index) => (
+                <div
+                  key={index}
+                  className={clsx(styles.itemWrapper, {
+                    [styles.alternate]: index % 2 === 0,
+                  })}
+                >
+                  {index % 2 !== 0 ? (
+                    <>
+                      <div
+                        className={clsx(
+                          styles.diagramItem,
+                          diagramItemThemeClass,
+                      
+                        )}
+                      >
+                        <img className={styles.diagramItem__img} src={useBaseUrl(item.imgSrc)} alt={item.title} />
+                      </div>
+                      <div
+                        className={clsx(styles.item, styles.evenItem, itemThemeClass)}
+                      >
+                        <div className={styles.item_content}>
+                          <div className={styles.item__title}>{item.title}</div>
+                          <div className={styles.item__subtitle}>{item.subtitle}</div>
+                        </div>
+                      </div>
+                    </>
+                  ) : (
+                    <>
+                      <div
+                        className={clsx(styles.item, styles.oddItem, itemThemeClass)}
+                      >
+                        <div className={clsx(styles.item_content)}>
+                          <div className={styles.item__title}>{item.title}</div>
+                          <div className={styles.item__subtitle}>{item.subtitle}</div>
+                        </div>
+                      </div>
+                      <div
+                        className={clsx(
+                          styles.diagramItem,
+                          diagramItemThemeClass,
+                      
+                        )}
+                      >
+                        <img className={styles.diagramItem__img} src={useBaseUrl(item.imgSrc)} alt={item.title} />
+                      </div>
+                    </>
+                  )}
+                </div>
+              ))}
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default Tiles;
diff --git a/docs-website/src/pages/solutions/_components/Tiles/tiles.module.scss b/docs-website/src/pages/solutions/_components/Tiles/tiles.module.scss
new file mode 100644
index 0000000000000..a71b0b8445541
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Tiles/tiles.module.scss
@@ -0,0 +1,203 @@
+.lightSection {
+  background: #FAFAFA;
+}
+
+.darkSection {
+  background: white;
+}
+
+.ecosystem_section {
+  padding: 4vh 0;
+  padding-bottom: 6vh;
+  display: flex;
+  justify-content: center;
+
+  .ecosystem_section_content {
+    width: 70%;
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    padding: 2rem 0rem;
+    text-align: left;
+
+    .ecosystem_section_upper_content {
+      padding-top: 1rem;
+      padding-left: 0; 
+      display: flex; 
+      justify-content: flex-start;
+
+      .ecosystem_section_heading {
+        line-height: 4rem;
+        font-family: Manrope;
+        font-family: Manrope;
+        font-size: 3.25rem;
+        font-style: normal;
+        font-weight: 500;
+        line-height: normal;
+        mix-blend-mode: luminosity;
+        width: 100%;
+      }
+    }
+
+    .ecosystem_section_lower_content {
+      margin-top: 48px;
+      display: flex;
+      flex-direction: row;
+      justify-content: center;
+
+      .itemWrappers {
+        display: flex;
+        flex-direction: column;
+        align-self: stretch; 
+
+        .itemWrapper {
+          display: flex;
+          padding: 4rem 0px;
+          width: 100%;
+          align-items: center;
+          justify-content: space-between;
+          flex-direction: row;
+          flex-wrap: nowrap;
+        }
+
+        .alternate {
+          .item {
+            margin-right: 2rem;
+            margin-left: 0;
+          }
+        }
+
+        .item {
+          display: flex;
+          justify-content: center;
+          align-items: flex-start;
+          padding: 0;
+          margin: 0;
+          margin-left: 4rem;
+          flex-grow: 1;
+          max-width: 520px;
+
+          .item__title {
+            font-family: Manrope;
+            font-size: 2.25rem;
+            font-style: normal;
+            font-weight: 500;
+            line-height: normal;
+            margin-bottom: 1rem;
+          }
+
+          .item__subtitle {
+            font-family: Manrope;
+            font-size: 1.25rem;
+            font-style: normal;
+            font-weight: 400;
+            line-height: normal;
+            color: #777E99;
+          }
+        }
+
+        .diagramItem {
+          display: flex;
+          overflow: hidden;
+          height: 400px;
+          min-width: 400px;
+          max-width: 400px;
+          justify-content: flex-end;
+          align-items: center;
+          border-radius: var(--number-scales-2s-20, 32px);
+          border: 0.5px solid #1890FF;
+          background: #FAFAFA;
+        }
+      }
+    }
+
+    .item, .diagramItem {
+      margin: 8px;
+    }
+  }
+}
+@media only screen and (max-width: 800px) {
+  .ecosystem_section {
+    padding: 1rem 0;
+    .ecosystem_section_content {
+      width: 90%;
+      min-width: 0;
+      height: auto !important;
+      padding-bottom: 48px;
+      padding-top: 48px;
+
+      .ecosystem_section_upper_content {
+        height: auto !important;
+
+        .ecosystem_section_heading {
+          font-size: 1.75rem;
+          line-height: 2.25rem;
+          font-weight: 600;
+          text-align: center;
+          width: 90%;
+          margin: auto;
+        }
+      }
+
+      .ecosystem_section_lower_content {
+        height: auto;
+        margin-top: 8px;
+
+        .diagramItem {
+          width: 100%;
+          height: auto;
+          border: none;
+        }
+        .itemWrappers {
+          .itemWrapper {
+            flex-direction: column-reverse;
+            &.alternate {
+              flex-direction: column;
+            }
+            width: 95vw;
+            padding: 2rem 0;
+            align-items: center; 
+            margin: auto;
+      
+          }
+      
+          .diagramItem {
+            height: auto;
+            width: 95vw;
+            aspect-ratio: 1;
+            min-width: auto;
+            margin: auto;
+          }
+      
+          .item {
+            text-align: center; 
+            padding : 1rem;
+            margin: 2rem 0!important;
+      
+            .item__title {
+              font-size: 1.5rem;
+            }
+            .item__subtitle {
+              font-size: 1rem;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+.darkDiagramItem {
+  background: linear-gradient(180deg, #F5EFF8 0%, #F1F4F4 100%);
+}
+
+.lightDiagramItem {
+  background: linear-gradient(37deg, rgba(238, 240, 242, 0.90) 2.81%, rgba(253, 249, 241, 0.90) 107.71%);
+}
+
+.diagramItem__img { 
+  overflow: hidden;
+  width: 100%;
+  height: auto;
+}
diff --git a/docs-website/src/pages/solutions/_components/Trials/index.js b/docs-website/src/pages/solutions/_components/Trials/index.js
new file mode 100644
index 0000000000000..ecca6685810b8
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Trials/index.js
@@ -0,0 +1,79 @@
+import React from "react";
+import styles from "./styles.module.scss";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+import clsx from "clsx";
+import Link from "@docusaurus/Link";
+
+const Trials = ({ onOpenTourModal, trialsContent }) => {
+  const { title, trialsCardItems } = trialsContent;
+
+  return (
+    <div className={styles.container}>
+      <div className={styles.trial}>
+        <div className={styles.trial_left}>
+          <div className={styles.left_content}>
+            <span className="">Discover. Observe. Govern.</span>
+            <p className="">{title}</p>
+            <div className={styles.btn_div}>
+              <Link to="/cloud">Book a Demo</Link>
+              <a onClick={onOpenTourModal}>Product Tour</a>
+            </div>
+            <Link className={styles.start_arrow} to="/docs">
+              Get started with Open Source →
+            </Link>
+          </div>
+        </div>
+        <div className={styles.trial_right}>
+          <div className={styles.right_content}>
+            <div className={styles.gradientTop} />
+            <div className={styles.gradientBottom} />
+            <div className={styles.right_l}>
+              {trialsCardItems.slice(0, 2).map((item, index) => (
+                <div
+                  key={index}
+                  className={clsx(styles[item.className])} 
+                >
+                  <img
+                    width={60}
+                    height={60}
+                    src={useBaseUrl(item.imgSrc)}
+                    alt={item.title}
+                  />
+                  {item.title.split("\n").map((line, idx) => (
+                    <React.Fragment key={idx}>
+                      {line}
+                      <br />
+                    </React.Fragment>
+                  ))}
+                </div>
+              ))}
+            </div>
+            <div className={styles.right_r}>
+              {trialsCardItems.slice(2).map((item, index) => (
+                <div
+                  key={index}
+                  className={clsx(styles[item.className])} 
+                >
+                  <img
+                    width={60}
+                    height={60}
+                    src={useBaseUrl(item.imgSrc)}
+                    alt={item.title}
+                  />
+                  {item.title.split("\n").map((line, idx) => (
+                    <React.Fragment key={idx}>
+                      {line}
+                      <br />
+                    </React.Fragment>
+                  ))}
+                </div>
+              ))}
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default Trials;
diff --git a/docs-website/src/pages/solutions/_components/Trials/styles.module.scss b/docs-website/src/pages/solutions/_components/Trials/styles.module.scss
new file mode 100644
index 0000000000000..9a991c4af36ab
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/Trials/styles.module.scss
@@ -0,0 +1,259 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  background: #fafafa;
+  width: 80vw;
+  min-width: 900px;
+  max-width: 1200px;
+  margin: 0 auto;
+}
+.trial {
+  height: 600px;
+  background: white;
+  display: flex;
+  justify-content: space-between;
+
+  .trial_left {
+    width: 55%;
+    height: 100%;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+
+    .left_content {
+      flex-grow: 1;
+      padding-right: 24px;
+
+      .trial_title {
+        font-weight: 300;
+      }
+
+      span {
+        color: #8088a3;
+        font-size: 1.5rem;
+        font-weight: 400;
+      }
+      p {
+        color: #2e2e38;
+        font-size: 3rem;
+        font-weight: 400;
+        line-height: normal;
+      }
+      .btn_div {
+        display: flex;
+        gap: 1rem;
+        margin-bottom: 1rem;
+
+        a:first-child {
+          cursor: pointer;
+          text-decoration: none;
+          display: inline-block;
+          font-size: 1rem;
+          background-color: #1890ff;
+          padding: 4px 20px;
+          border-radius: 50px;
+          margin: 0 0 0 0;
+          color: white;
+          transition: opacity .2s ease-in-out;
+          &:hover {
+            opacity: .9;
+          }
+        }
+
+        a:nth-child(2) {
+          cursor: pointer;
+          text-decoration: none;
+          display: inline-block;
+          font-size: 1rem;
+          padding: 4px 20px;
+          border-radius: 50px;
+          margin: 0 0 0 0;
+          background-color: transparent;
+          color: #1890ff;
+          border: 1px solid #1890ff;
+          transition: background-color .2s ease-in-out;
+          &:hover {
+            background-color: #1890ff1A;
+          }
+        }
+      }
+
+      .start_arrow {
+        margin-top: .5rem;
+        display: inline-block;
+        font-size: 1.1rem;
+        color: #1890ff;
+        font-weight: 500;
+        cursor: pointer;
+        text-decoration: none;
+      }
+    }
+  }
+  .trial_right {
+    width: 45%;
+    max-width: 520px;
+    height: 100%;
+    position: relative;
+    overflow: hidden;
+
+    .gradientTop, .gradientBottom {
+      position: absolute;
+      background: linear-gradient(#FFFFFF, #FFFFFF00);
+      height: 100px;
+      width: 100%;
+      z-index: 10;
+    }
+    .gradientBottom {
+      transform: rotate(180deg);
+      bottom: 0;
+    }
+
+    .right_content {
+      height: 100%;
+      display: flex;
+      gap: 2rem;
+      .right_l {
+        width: 50%;
+        height: 95%;
+        margin-top: -.5rem;
+        display: flex;
+        flex-direction: column;
+        gap: 1rem;
+
+        div {
+          font-size: 1.25rem;
+          font-weight: 400;
+          color: #000;
+          line-height: 1.5rem;
+          img {
+            margin-bottom: 8px;
+          }
+        }
+
+        div:first-child {
+          display: flex;
+          flex-direction: column;
+          justify-content: center;
+          padding-left: 2rem;
+          padding-right: 1.5rem;
+          height: 49%;
+          border-radius: 35px;
+          background-color: #fff;
+          border: 1px solid #ddd;
+          // background-image: linear-gradient(to bottom, #FC526333, #FC5263);
+        }
+
+        div:nth-child(2) {
+          display: flex;
+          flex-direction: column;
+          justify-content: center;
+          padding-left: 2rem;
+          padding-right: 1.5rem;
+          background-color: #fff;
+          border: 1px solid #ddd;
+          // background-image: linear-gradient(
+          //   to bottom,
+          //   #77B750,
+          //   #77B75080
+          // );
+          height: 49%;
+          border-radius: 35px;
+        }
+      }
+      .right_r {
+        width: 50%;
+        height: calc(100% - 2rem);
+        margin-top: 2.5rem;
+        display: flex;
+        flex-direction: column;
+        gap: 1rem;
+
+        div {
+          font-size: 1.25rem;
+          font-weight: 400;
+          color: #000;
+          line-height: 1.5rem;
+          img {
+            margin-bottom: 8px;
+          }
+        }
+
+        div:first-child {
+          display: flex;
+          flex-direction: column;
+          justify-content: center;
+          padding-left: 2rem;
+          padding-right: 1.5rem;
+          // background-image: linear-gradient(to bottom, #1890FF80, #1890FF);
+          background-color: #fff;
+          border: 1px solid #ddd;
+          height: 49%;
+          border-radius: 35px;
+        }
+
+        div:nth-child(2) {
+          display: flex;
+          flex-direction: column;
+          justify-content: center;
+          padding-left: 2rem;
+          padding-right: 1.5rem;
+          background-color: #fff;
+          border: 1px solid #ddd;
+          // background-image: linear-gradient(
+          //   to bottom,
+          //   #EFB300,
+          //   #EFB30033
+          // );
+          height: 49%;
+          border-radius: 35px;
+        }
+      }
+    }
+  }
+}
+
+// Responsiveness
+@media (max-width: 800px) {
+
+  .container {
+    flex-direction: column;
+    width: 90vw;
+    min-width: 0;
+    margin: 4rem auto;
+  }
+  .trial {
+    flex-direction: column;
+    justify-content: flex-start;
+    height: auto;
+    .trial_left {
+      width: 100%;
+      .left_content {
+        span {
+          font-size: 1.3rem;
+        }
+        p {
+          font-size: 2.5rem;
+        }
+      }
+      .start_arrow {
+        margin-top: 0;
+        margin-bottom: 1rem;
+      }
+    }
+    .trial_right {
+      width: 100%;
+      max-width: none;
+      height: 500px;
+      margin: auto;
+    }
+    .right_content {
+      .right_l, .right_r {
+        div {
+          font-size: .9rem!important;
+          line-height: 1.2rem !important;
+          color: #333!important;
+        }
+      }
+    }
+  }
+}
diff --git a/docs-website/src/pages/solutions/_components/UnifiedTabs/index.js b/docs-website/src/pages/solutions/_components/UnifiedTabs/index.js
new file mode 100644
index 0000000000000..0ccf0501670a3
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/UnifiedTabs/index.js
@@ -0,0 +1,42 @@
+import React, { useState } from 'react';
+import styles from './styles.module.scss';
+import clsx from 'clsx';
+import useBaseUrl from '@docusaurus/useBaseUrl';
+
+const TabbedComponent = ({ unifiedTabsData }) => {
+  const [activeTab, setActiveTab] = useState(0);
+
+  return (
+    <div className={clsx(styles.tabbedComponent)}>
+      <div className={clsx(styles.tabsContainer)}>
+        <div className={clsx(styles.tabs)}>
+          {unifiedTabsData.map((tab, index) => (
+            <div
+              key={index}
+              className={clsx(styles.tabButton, { [styles.active]: activeTab === index })}
+              onClick={() => setActiveTab(index)}
+            >
+              <div className={clsx(styles.tabButtonText, { [styles.active]: activeTab === index })}>
+                {tab.tabName}
+              </div>
+            </div>
+          ))}
+        </div>
+      </div>
+      <div className={clsx(styles.container)}>
+        <div className={clsx(styles.tabContent)}>
+          <div className={styles.tabTitle}>{unifiedTabsData[activeTab].title}</div>
+          <div className={styles.tabDescription}>{unifiedTabsData[activeTab].description}</div>
+        </div>
+
+        <div className={clsx(styles.imageContainer)}>
+          <div className={clsx(styles.tabImage)}>
+            <img src={unifiedTabsData[activeTab].image} alt={unifiedTabsData[activeTab].imagetabName} />
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default TabbedComponent;
diff --git a/docs-website/src/pages/solutions/_components/UnifiedTabs/styles.module.scss b/docs-website/src/pages/solutions/_components/UnifiedTabs/styles.module.scss
new file mode 100644
index 0000000000000..a3a1a455af3d3
--- /dev/null
+++ b/docs-website/src/pages/solutions/_components/UnifiedTabs/styles.module.scss
@@ -0,0 +1,124 @@
+.tabbedComponent {
+  text-align: left;
+  padding: 20px;
+  padding-top: 48px;
+  padding-bottom: 48px;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.tabsContainer {
+  width: 100%; // Ensure full width
+  max-width: 1200px; // Limit to the same max-width as content
+  margin-bottom: 1rem; 
+  display: flex;
+}
+
+.tabs {
+  display: flex;
+  justify-content: space-between; // Space out tabs across the container
+  width: 100%; // Ensure tabs take full width
+}
+
+.tabButton {
+  flex: 1; 
+  padding: 0.8rem 1.5rem; 
+  background: none;
+  border: none;
+  cursor: pointer;
+  font-size: 1.5rem;
+  font-weight: 500;
+  color: #777e99;
+  text-align: center;
+
+
+  .tabButtonText {
+    width: min-content;
+    margin: auto;
+    transition: color 0.2s, border-bottom 0.2s;
+  }
+  .tabButtonText:hover, .active {
+    color: #1890ff;
+    border-bottom: 3px solid #1890ff;
+  }
+}
+
+.container {
+  display: flex;
+  flex-direction: row;
+  background: white;
+  max-width: 1200px; // Same max-width as tabs
+  width: 100%; // Full width to align with tabs
+  overflow: hidden;
+}
+
+.tabContent {
+  flex: 1;
+  padding: 2rem;
+  margin: auto;
+}
+
+.tabTitle {
+  font-size: 2.5rem;
+  font-weight: 500;
+  color: #373A47;
+  line-height: 120%;
+  margin-bottom: 1rem;
+}
+
+.tabTitle.active {
+  color: #1890ff;
+  text-decoration: underline;
+}
+
+.tabDescription {
+  font-size: 1.4rem;
+  color: #777e99;
+  line-height: 2rem;
+  font-weight: 300;
+}
+
+.imageContainer {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  margin: 1rem 0;
+  width: 30rem;
+}
+
+.tabImage {
+  width: 100%;
+  height: 100%;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  box-shadow: 0px 2px 4px 0px #0000001C;
+  border-radius: 2.2rem;
+}
+
+@media (min-width: 768px) {
+  .container {
+    flex-direction: row;
+    padding: 40px 32px;
+  }
+
+  .imageContainer {
+    margin: 1rem;
+  }
+}
+
+@media (max-width: 768px) {
+  .container {
+    display: block;
+  }
+
+  .imageContainer {
+    margin: auto;
+    max-width: 100%;
+  }
+
+  .tabTitle {
+    font-size: 1.8rem;
+  }
+}
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/discovery/_content/discoveryCaseStudyContent.js b/docs-website/src/pages/solutions/discovery/_content/discoveryCaseStudyContent.js
new file mode 100644
index 0000000000000..95bbfde0bd12a
--- /dev/null
+++ b/docs-website/src/pages/solutions/discovery/_content/discoveryCaseStudyContent.js
@@ -0,0 +1,26 @@
+const caseStudyContent = {
+      title: "See how industry leaders are using DataHub for Discovery today.",
+      backgroundColor: "#F3F3F6",
+      items: [
+        {
+          imgSrc: "/img/solutions/logo-notion.png",
+          title: "How Notion Used DataHub to harness their sprawling data.",
+          link: "https://www.notion.so/blog/a-brief-history-of-notions-data-catalog",
+          alt: "notion"
+        },
+        {
+          imgSrc: "/img/solutions/logo-myob.png",
+          title: "How MYOB eliminated breaking changes with DataHub.",
+          link: "/adoption-stories/#myob",
+          alt: "MYOB",
+        },
+        {
+          imgSrc: "/img/solutions/logo-dpg-media.png",
+          title: "How Acryl Data Helped DPG Media Save 25% Per Mo in Snowflake.",
+          link: "/adoption-stories/#dpg-media",
+          alt: "DPG Media",
+        }
+      ]
+    };
+  
+export default caseStudyContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/discovery/_content/discoveryHeroContent.js b/docs-website/src/pages/solutions/discovery/_content/discoveryHeroContent.js
new file mode 100644
index 0000000000000..085e94085126b
--- /dev/null
+++ b/docs-website/src/pages/solutions/discovery/_content/discoveryHeroContent.js
@@ -0,0 +1,8 @@
+const heroContent = {
+      topQuote: "Discovery in DATAHUB",
+      title: "Make data \n\n democratization a reality",
+      description: "Enable everyone in your organization to effortlessly discover trustworthy data, tailor experiences for each persona, eliminate breaking changes with lineage, and build confidence in your data with a unified view of business and technical context.",
+      imgSrc: "/img/solutions/hero-discovery.png",
+    };
+  
+  export default heroContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/discovery/_content/discoveryQuickstartContent.js b/docs-website/src/pages/solutions/discovery/_content/discoveryQuickstartContent.js
new file mode 100644
index 0000000000000..e1c6a84512fd5
--- /dev/null
+++ b/docs-website/src/pages/solutions/discovery/_content/discoveryQuickstartContent.js
@@ -0,0 +1,16 @@
+const quickstartData = [
+  {
+    title: "Where can I find our quarterly revenue reporting?",
+    image: "/img/solutions/icon-revenue.png",
+  },
+  {
+    title: "This metric looks wrong. How was it calculated?",
+    image: "/img/solutions/icon-metric.png", 
+  },
+  {
+    title: "What reports will be impacted during a data migration?",
+    image: "/img/solutions/icon-migration.png",
+  }
+];
+
+export default quickstartData;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/discovery/_content/discoveryTestimonialsContent.js b/docs-website/src/pages/solutions/discovery/_content/discoveryTestimonialsContent.js
new file mode 100644
index 0000000000000..e9c72175ffcb7
--- /dev/null
+++ b/docs-website/src/pages/solutions/discovery/_content/discoveryTestimonialsContent.js
@@ -0,0 +1,10 @@
+const testimonialsData = {
+  title: "Enter end-to-end Data Discovery.",
+  feature1: "Data Observability",
+  feature1Link: "/solutions/observability",
+  feature2: "Governance",
+  feature2Link: "/solutions/governance",
+  imgSrc: "/img/solutions/discovery-icons-group.png",
+};
+
+export default testimonialsData;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/discovery/_content/discoveryTilesContent.js b/docs-website/src/pages/solutions/discovery/_content/discoveryTilesContent.js
new file mode 100644
index 0000000000000..c15042ea1c8db
--- /dev/null
+++ b/docs-website/src/pages/solutions/discovery/_content/discoveryTilesContent.js
@@ -0,0 +1,67 @@
+const tilesContent = [
+  {
+      title: "Enable self-service <br/> data discovery.",
+      theme: "dark",
+      tileItems: [
+        {
+          title: "Your role, your view: discover data that matters to you.",
+          subtitle: "Tailor search experiences for every user type, from analysts to executives. Foster company-wide engagement and turn every employee into a data champion.",
+          imgSrc: "/img/solutions/discovery-tile-1.png",
+        },
+        {
+          title: "Silence the irrelevant. Amplify what counts.",
+          subtitle: "Cut through data clutter to reveal the assets that truly move the needle for your organization.",
+          imgSrc: "/img/solutions/discovery-tile-2.png",
+        },
+        {
+          title: "Maximize relevance. Minimize time-to-discovery.",
+          subtitle: "Tailor search results to reflect your organization's trust signals. Boost user confidence by prioritizing results that meet your standards of reliability and relevance.",
+          imgSrc: "/img/solutions/discovery-tile-3.png",
+        }
+      ]
+    },
+    {
+      title: "Unlock the full potential of<br/>automated data lineage.",
+      theme: "light",
+      tileItems: [
+        {
+          title: "Demystify complex, cross-platform dependency chains.",
+          subtitle: "Find out when things go wrong, with alerts that reach your team where they work—whether it’s Slack, email, or anywhere else.",
+          imgSrc: "/img/solutions/discovery-tile-4.png",
+        },
+        {
+          title: "Know your data’s Impact.",
+          subtitle: "Instantly identify downstream consumers of your data. Enable seamless communication and collaboration across your data ecosystem.",
+          imgSrc: "/img/solutions/discovery-tile-5.png",
+        },
+        {
+          title: "Illuminate the black box of data transformations.",
+          subtitle: "Shine a light on how your key metrics are derived. Automated lineage provides transparency, fostering trust in your data-driven decisions.",
+          imgSrc: "/img/solutions/discovery-tile-6.png",
+        }
+      ]
+    },
+    {
+      title: "Build trust in the relevance<br/>and accuracy of your data.",
+      theme: "dark",
+      tileItems: [
+        {
+          title: "Increase your data confidence.",
+          subtitle: "Tailor search experiences for every user type, from analysts to executives. Foster company-wide engagement and turn every employee into a data champion.",
+          imgSrc: "/img/solutions/discovery-tile-7.png",
+        },
+        {
+          title: "Time travel through your data's evolution.",
+          subtitle: "View the shape and content of your data as it changes over time. Gain confidence in your current data by understanding its past.",
+          imgSrc: "/img/solutions/discovery-tile-8.png",
+        },
+        {
+          title: "Your data quality companion, everywhere you work.",
+          subtitle: "Seamlessly integrate DataHub's insights into your BI tools and communication channels. Keep data trust at the forefront of every decision.",
+          imgSrc: "/img/solutions/discovery-tile-9.png",
+        }
+      ]
+    }
+]
+  
+  export default tilesContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/discovery/_content/discoveryTrialsContent.js b/docs-website/src/pages/solutions/discovery/_content/discoveryTrialsContent.js
new file mode 100644
index 0000000000000..b9b695549210d
--- /dev/null
+++ b/docs-website/src/pages/solutions/discovery/_content/discoveryTrialsContent.js
@@ -0,0 +1,24 @@
+const trialsContent = {
+  title: "Make data democratization a reality today.",
+  trialsCardItems: [
+    {
+      title: "Unlock self-service data discovery.",
+      imgSrc: "/img/solutions/trial-icon-lock.svg",
+      
+    },
+    {
+      title: "Stop breaking changes before they happen.",
+      imgSrc: "/img/solutions/trial-icon-alert.svg",
+    },
+    {
+      title: "Build trust in the relevance and accuracy of your data.",
+      imgSrc: "/img/solutions/trial-icon-star.svg",
+    },
+    {
+      title: "Unify Discovery, Observability and Governance in one tool.",
+      imgSrc: "/img/solutions/trial-icon-link.svg",
+    }
+  ]
+};
+
+export default trialsContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/discovery/index.js b/docs-website/src/pages/solutions/discovery/index.js
new file mode 100644
index 0000000000000..01efca1e0e7c6
--- /dev/null
+++ b/docs-website/src/pages/solutions/discovery/index.js
@@ -0,0 +1,62 @@
+import React, { useState } from "react";
+import Layout from "@theme/Layout";
+import useDocusaurusContext from "@docusaurus/useDocusaurusContext";
+import Hero from "../_components/Hero";
+import QuickstartContent from "../_components/QuickstartContent";
+import Tiles from "../_components/Tiles";
+import Trials from "../_components/Trials";
+import Testimonials from "../_components/Testimonials";
+import CaseStudy from "../_components/CaseStudy";
+import CloseButton from "@ant-design/icons/CloseCircleFilled";
+import quickstartData from "./_content/discoveryQuickstartContent";
+import heroContent from "./_content/discoveryHeroContent";
+import caseStudyContent from "./_content/discoveryCaseStudyContent";
+import Integrations from "../_components/Integrations";
+import tilesContent from "./_content/discoveryTilesContent";
+import testimonialsData from "./_content/discoveryTestimonialsContent";
+import trialsContent from "./_content/discoveryTrialsContent";
+
+function Home() {
+  const context = useDocusaurusContext();
+  const { siteConfig = {} } = context;
+
+  if (siteConfig.customFields.isSaas) {
+    window.location.replace("/docs");
+  }
+
+  const [isTourModalVisible, setIsTourModalVisible] = useState(false);
+  const onOpenTourModal = () => {
+    setIsTourModalVisible(true);
+  };
+  const onCloseTourModal = () => {
+    setIsTourModalVisible(false);
+  };
+  return !siteConfig.customFields.isSaas ? (
+    <Layout
+      title={siteConfig.tagline}
+      description="DataHub is a data discovery application built on an extensible data catalog that helps you tame the complexity of diverse data ecosystems."
+    >
+      {isTourModalVisible ? (
+        <div className="tourModal">
+          <div className="closeButtonWrapper" onClick={onCloseTourModal}>
+            <CloseButton />
+          </div>
+          <iframe src="https://www.acryldata.io/tour" />
+        </div>
+      ) : null}
+      <Hero onOpenTourModal={onOpenTourModal} heroContent={heroContent}/>
+      <Integrations />
+      <QuickstartContent quickstartContent={quickstartData} />
+      <Testimonials testimonialsData={testimonialsData} />
+      <div>
+        {tilesContent.map((content, index) => (
+          <Tiles key={index} tilesContent={content} />
+        ))}
+      </div>
+      <CaseStudy caseStudyContent= {caseStudyContent}/>
+      <Trials onOpenTourModal={onOpenTourModal} trialsContent={trialsContent} />
+    </Layout>
+  ) : null;
+}
+
+export default Home;
diff --git a/docs-website/src/pages/solutions/discovery/styles.module.scss b/docs-website/src/pages/solutions/discovery/styles.module.scss
new file mode 100644
index 0000000000000..af0cd90c99134
--- /dev/null
+++ b/docs-website/src/pages/solutions/discovery/styles.module.scss
@@ -0,0 +1,21 @@
+.container {
+    display: flex;
+    flex-direction: column;
+    background: #fafafa;
+    width: 80vw;
+    min-width: 900px;
+    max-width: 1200px;
+    margin: 0 auto;
+  }
+
+  // Responsiveness
+  @media (max-width: 800px) {
+  
+    .container {
+      flex-direction: column;
+      width: 90vw;
+      min-width: 0;
+      margin: 4rem auto;
+    }
+  }
+  
diff --git a/docs-website/src/pages/solutions/governance/_content/governanceCaseStudyContent.js b/docs-website/src/pages/solutions/governance/_content/governanceCaseStudyContent.js
new file mode 100644
index 0000000000000..4e12756a980c9
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/_content/governanceCaseStudyContent.js
@@ -0,0 +1,26 @@
+const caseStudyContent = {
+      title: "See how industry leaders are using DataHub for Governance today.",
+      backgroundColor: "#FFFFFF",
+      items: [
+        {
+          imgSrc: "/img/solutions/optum.jpg",
+          title: "How Optum uses DataHub to govern and manage access.",
+          link: "/adoption-stories/#optum",
+          alt: "optum"
+        },
+        {
+          imgSrc: "/img/solutions/checkout.jpg",
+          title: "How Checkout.com manages compliance with sensitive data.",
+          link: "/adoption-stories/#checkout-com",
+          alt: "Checkout.com",
+        },
+        {
+          imgSrc: "/img/solutions/wolt.png",
+          title: "How Wolt complies with legal requirements using DataHub.",
+          link: "/adoption-stories/#wolt",
+          alt: "Wolt",
+        }
+      ]
+    };
+  
+export default caseStudyContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/governance/_content/governanceHeroContent.js b/docs-website/src/pages/solutions/governance/_content/governanceHeroContent.js
new file mode 100644
index 0000000000000..779a9cb7062ff
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/_content/governanceHeroContent.js
@@ -0,0 +1,8 @@
+const heroContent = {
+      topQuote: "Data governance in datahub",
+      title: "Measure and minimize compliance risk, effortlessly.",
+      description: "Ensure every data asset is accounted for and responsibility governed by defining and enforcing documentation standards Automate your governance program to automatically classify assets as they evolve over time. Minimize redundant, manual work with GenAI documentation, AI-driven classification, smart propagation, and more.",
+      imgSrc: "/img/solutions/hero-governance.png",
+    };
+  
+  export default heroContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/governance/_content/governancePersonaContent.js b/docs-website/src/pages/solutions/governance/_content/governancePersonaContent.js
new file mode 100644
index 0000000000000..9d456bee5d30a
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/_content/governancePersonaContent.js
@@ -0,0 +1,31 @@
+const personaContent = {
+      title: "The only platform that serves everyone involved with Governance.",
+      personas: [
+        {
+          imgSrc: "/img/solutions/persona_compliance_officers.png",
+          feature1: "Data Classification Glossary",
+          feature2: "Custom Form builder",
+          feature3: "Automated Form Assigning",
+          feature4: "Analytics Reporting",
+          alt: "governance_officers"
+        },
+        {
+          imgSrc: "/img/solutions/persona_developers.png",
+          feature1: "Highly Extensible Open-Core",
+          feature2: "Rich APIs and SDKs",
+          feature3: "Automation Framework",
+          feature4: "Shift Left Architecture",
+          alt: "developers",
+        },
+        {
+          imgSrc: "/img/solutions/persona_owners_and_smes.png",
+          feature1: "Seamless UI Workflows",
+          feature2: "Respond to Requests",
+          feature3: "Notifications Where They Work",
+          feature4: "Proposal Workflows",
+          alt: "data_owners",
+        }
+      ]
+    };
+  
+export default personaContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/governance/_content/governanceQuickstartContent.js b/docs-website/src/pages/solutions/governance/_content/governanceQuickstartContent.js
new file mode 100644
index 0000000000000..c59e3c215d707
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/_content/governanceQuickstartContent.js
@@ -0,0 +1,12 @@
+const quickstartData = [
+  {
+    title: "Running into challenges while tracking sensitive data as it flows through your data landscape?",
+    image: "/img/solutions/icon-cloud.png",
+  },
+  {
+    title: "Is enforcing and measuring compliance initiatives impossibly manual, tedious and slow?",
+    image: "/img/solutions/icon-calendar.png", 
+  },
+];
+
+export default quickstartData;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/governance/_content/governanceTestimonialsContent.js b/docs-website/src/pages/solutions/governance/_content/governanceTestimonialsContent.js
new file mode 100644
index 0000000000000..a450c20926bd5
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/_content/governanceTestimonialsContent.js
@@ -0,0 +1,10 @@
+const testimonialsData = {
+  title: "Enter end-to-end Data Governance.",
+  feature1: "Data Discovery",
+  feature2: "Observability",
+  feature1Link: "/solutions/discovery",
+  feature2Link: "/solutions/observability",
+  imgSrc: "/img/solutions/governance-icons-group.png",
+};
+
+export default testimonialsData;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/governance/_content/governanceTilesContent.js b/docs-website/src/pages/solutions/governance/_content/governanceTilesContent.js
new file mode 100644
index 0000000000000..d8786f9d0c8ab
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/_content/governanceTilesContent.js
@@ -0,0 +1,51 @@
+const tilesContent = [
+  {
+      title: "Save countless hours of manual <br /> labeling with automated classification.",
+      theme: "dark",
+      tileItems: [
+        {
+          title: "Data producers classify, DataHub centralizes.",
+          subtitle: "Tag sensitive data where it's born. See classifications automatically reflected in DataHub, ensuring consistent compliance across your data ecosystem.",
+          imgSrc: "/img/solutions/governance-tile-1.png",
+        },
+        {
+          title: "Classification that flows: Lineage-driven compliance.",
+          subtitle: "Automatically classify your data as it moves and transforms. Ensure consistent compliance classification across your entire data ecosystem.",
+          imgSrc: "/img/solutions/governance-tile-2.png",
+        },
+        {
+          title: "Keep your tags in harmony, automatically.",
+          subtitle: "Create a seamless flow of metadata between DataHub and source systems, ensuring tags are always up-to-date, everywhere.'s trust signals. Boost user confidence by prioritizing results that meet your standards of reliability and relevance.",
+          imgSrc: "/img/solutions/governance-tile-3.png",
+        },
+        {
+          title: "Metadata gaps? AI has you covered.",
+          subtitle: "Combine human insight with Gen AI prowess to enhance accuracy and coverage in your metadata management.",
+          imgSrc: "/img/solutions/governance-tile-4.png",
+        }
+      ]
+    },
+    {
+      title: "Streamline your compliance program: <br />Standardize, delegate, and measure with ease.",
+      theme: "light",
+      tileItems: [
+        {
+          title: "Customize your compliance blueprint.",
+          subtitle: "Define compliance requirements to match your organization’s unique needs. Establish clear annotation guidelines to ensure consistency and accuracy across your data landscape.",
+          imgSrc: "/img/solutions/governance-tile-5.png",
+        },
+        {
+          title: "Equip data experts with crystal-clear guidelines.",
+          subtitle: "Route annotation tasks to those who know the data best. Take the guesswork out of compliance, freeing up time for innovation.",
+          imgSrc: "/img/solutions/governance-tile-6.png",
+        },
+        {
+          title: "Never lose sight of compliance goals.",
+          subtitle: "Maintain a clear view of progress and keep data experts accountable with targeted alerts. Transform compliance tracking from a chore to a breeze",
+          imgSrc: "/img/solutions/governance-tile-7.png",
+        }
+      ]
+    }
+]
+  
+  export default tilesContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/governance/_content/governanceTrialsContent.js b/docs-website/src/pages/solutions/governance/_content/governanceTrialsContent.js
new file mode 100644
index 0000000000000..a1eed676e8909
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/_content/governanceTrialsContent.js
@@ -0,0 +1,27 @@
+const trialsContent = {
+  title: "Start measuring and minimizing your compliance risk today.",
+  trialsCardItems: [
+    {
+      title: "Get everyone speaking the same data language.",
+      imgSrc: "/img/solutions/trial-icon-language.png",
+      className: "soc",  
+    },
+    {
+      title: "Empower data owners to own assets, with minimal effort.",
+      imgSrc: "/img/solutions/trial-icon-owner.png",
+      className: "cost", 
+    },
+    {
+      title: "Automated classification moves at the speed of your data.",
+      imgSrc: "/img/solutions/trial-icon-lightening.png",
+      className: "enterprise", 
+    },
+    {
+      title: "Monitor and enforce governance standards.",
+      imgSrc: "/img/solutions/trial-icon-standard.png",
+      className: "link", 
+    },
+  ],
+};
+
+export default trialsContent;
diff --git a/docs-website/src/pages/solutions/governance/index.js b/docs-website/src/pages/solutions/governance/index.js
new file mode 100644
index 0000000000000..82a45cef83438
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/index.js
@@ -0,0 +1,66 @@
+import React, { useState } from "react";
+import Layout from "@theme/Layout";
+import useDocusaurusContext from "@docusaurus/useDocusaurusContext";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+import Hero from "../_components/Hero";
+import QuickstartContent from "../_components/QuickstartContent";
+import Tiles from "../_components/Tiles";
+import Testimonials from "../_components/Testimonials";
+import CaseStudy from "../_components/CaseStudy";
+import Persona from "../_components/Persona";
+import styles from "./styles.module.scss";
+import CloseButton from "@ant-design/icons/CloseCircleFilled";
+import Link from "@docusaurus/Link";
+import quickstartData from "./_content/governanceQuickstartContent";
+import heroContent from "./_content/governanceHeroContent";
+import caseStudyContent from "./_content/governanceCaseStudyContent";
+import personaContent from "./_content/governancePersonaContent";
+import tilesContent from "./_content/governanceTilesContent";
+import testimonialsData from "./_content/governanceTestimonialsContent";
+import trialsContent from "./_content/governanceTrialsContent";
+import Trials from "../_components/Trials";
+
+function Home() {
+  const context = useDocusaurusContext();
+  const { siteConfig = {} } = context;
+
+  if (siteConfig.customFields.isSaas) {
+    window.location.replace("/docs");
+  }
+
+  const [isTourModalVisible, setIsTourModalVisible] = useState(false);
+  const onOpenTourModal = () => {
+    setIsTourModalVisible(true);
+  };
+  const onCloseTourModal = () => {
+    setIsTourModalVisible(false);
+  };
+  return !siteConfig.customFields.isSaas ? (
+    <Layout
+      title={siteConfig.tagline}
+      description="DataHub is a data discovery application built on an extensible data catalog that helps you tame the complexity of diverse data ecosystems."
+    >
+      {isTourModalVisible ? (
+        <div className="tourModal">
+          <div className="closeButtonWrapper" onClick={onCloseTourModal}>
+            <CloseButton />
+          </div>
+          <iframe src="https://www.acryldata.io/tour" />
+        </div>
+      ) : null}
+      <Hero onOpenTourModal={onOpenTourModal} heroContent={heroContent}/>
+      <Persona personaContent={personaContent} />
+      <QuickstartContent quickstartContent={quickstartData} />
+      <Testimonials testimonialsData={testimonialsData} />
+      <div>
+        {tilesContent.map((content, index) => (
+          <Tiles key={index} tilesContent={content} />
+        ))}
+      </div>
+      <CaseStudy caseStudyContent= {caseStudyContent}/>
+      <Trials onOpenTourModal={onOpenTourModal} trialsContent={trialsContent} />
+    </Layout>
+  ) : null;
+}
+
+export default Home;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/governance/styles.module.scss b/docs-website/src/pages/solutions/governance/styles.module.scss
new file mode 100644
index 0000000000000..a26d953e9e5ad
--- /dev/null
+++ b/docs-website/src/pages/solutions/governance/styles.module.scss
@@ -0,0 +1,21 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  background: #fafafa;
+  width: 80vw;
+  min-width: 900px;
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+// Responsiveness
+@media (max-width: 800px) {
+
+  .container {
+    flex-direction: column;
+    width: 90vw;
+    min-width: 0;
+    margin: 4rem auto;
+  }
+}
+
diff --git a/docs-website/src/pages/solutions/observability/_content/observeCaseStudyContent.js b/docs-website/src/pages/solutions/observability/_content/observeCaseStudyContent.js
new file mode 100644
index 0000000000000..ca263fe0d7ff9
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/_content/observeCaseStudyContent.js
@@ -0,0 +1,26 @@
+const caseStudyContent = {
+      title: "See how industry leaders are using DataHub for Observability today.",
+      backgroundColor: "#F3F3F6",
+      items: [
+        {
+          imgSrc: "/img/solutions/logo-notion.png",
+          title: "How Notion Uses DataHub Cloud to Ensure Data Reliability.",
+          link: "/adoption-stories/#notion",
+          alt: "notion"
+        },
+        {
+          imgSrc: "/img/solutions/logo-myob.png",
+          title: "How MYOB Improved Data Reliability for dbt and Snowflake.",
+          link: "/adoption-stories/#myob",
+          alt: "MYOB",
+        },
+        {
+          imgSrc: "/img/solutions/miro.png",
+          title: "How DataHub Helped Miro improve Data Product Reliability.",
+          link: "/adoption-stories/#miro",
+          alt: "Miro",
+        }
+      ]
+    };
+  
+export default caseStudyContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/observability/_content/observeHeroContent.js b/docs-website/src/pages/solutions/observability/_content/observeHeroContent.js
new file mode 100644
index 0000000000000..895b2ca13de8b
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/_content/observeHeroContent.js
@@ -0,0 +1,8 @@
+const heroContent = {
+      topQuote: "OBSERVABILITY in DATAHUB CLOUD",
+      title: "Trust your most important data.",
+      description: "Detect, resolve, and prevent data quality issues before they impact your business. Seamlessly integrate Data Quality, Data Discovery and Data Governance in one place.",
+      imgSrc: "/img/solutions/hero-observe.png",
+    };
+  
+  export default heroContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/observability/_content/observeQuickstartContent.js b/docs-website/src/pages/solutions/observability/_content/observeQuickstartContent.js
new file mode 100644
index 0000000000000..75c4a78a7390e
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/_content/observeQuickstartContent.js
@@ -0,0 +1,16 @@
+const quickstartData = [
+  {
+    title: "Constantly getting messages about broken data?",
+    image: "/img/solutions/icon-cloud.png",
+  },
+  {
+    title: "Tired of spending days cleaning up bad data instead of making forward progress?",
+    image: "/img/solutions/icon-calendar.png", 
+  },
+  {
+    title: "Having trouble managing different tools for data quality, discovery, and compliance?",
+    image: "/img/solutions/icon-wrench.png",
+  }
+];
+
+export default quickstartData;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/observability/_content/observeResourceContent.js b/docs-website/src/pages/solutions/observability/_content/observeResourceContent.js
new file mode 100644
index 0000000000000..512a1a80ff4b6
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/_content/observeResourceContent.js
@@ -0,0 +1,61 @@
+const resourceData = [
+  {
+    title: "Introducing Acryl Observe",
+    tag: "Blog",
+    backgroundImage:
+      "https://cdn.sanity.io/images/cqo9wkgf/production/710b727fd06db2bbc892e4e9a3f403513fdd75fa-2000x1413.png?w=3840&q=75&fit=clip&auto=format",
+    link: "https://www.acryldata.io/blog/data-quality-should-be-part-of-the-data-catalog-introducing-acryl-observe", 
+    readTime: "4"
+  },
+  {
+    title: "When Data Quality Fires Break Out",
+    tag: "Blog",
+    backgroundImage:
+      "https://cdn.sanity.io/images/cqo9wkgf/production/039b82490a6322fde04326d892494923ea45a957-599x624.png?w=1200&q=75&fit=clip&auto=format",
+    link: "https://www.acryldata.io/blog/always-be-the-first-to-know-with-acryl-observe",
+    readTime: "5"
+
+  },
+  {
+    title: "Five Signs You Need a Unified Data Observability Solution",
+    tag: "Checklist",
+    backgroundImage:
+      "https://cdn.sanity.io/images/cqo9wkgf/production/3dab61d2fe39b89d2c32f8399e75a699de3da76c-1920x936.webp?w=3840&q=75&fit=clip&auto=format",
+    link: "https://www.acryldata.io/blog/five-signs-you-need-a-data-observability-solution",
+    readTime: "4",
+  },
+  {
+    title: "Preventing Data Delays with Acryl Observe",
+    tag: "Blog",
+    backgroundImage:
+      "https://cdn.sanity.io/images/cqo9wkgf/production/375b6e03a6a113e43eaa13ffda8072c07ca69c87-611x693.png?w=1920&q=75&fit=clip&auto=format",
+    link: "https://www.acryldata.io/blog/preventing-data-freshness-problems-with-acryl-observe",
+    readTime: "6",
+  },
+  {
+    title: "Detecting Unexpected Table Volume Changes with Acryl Observe",
+    tag: "Blog",
+    backgroundImage:
+      "https://cdn.sanity.io/images/cqo9wkgf/production/e4f8d8ab0736631f968b1fdd8411789d2fb2af7f-2000x1730.png?w=3840&q=75&fit=clip&auto=format",
+    link: "https://www.acryldata.io/blog/product-update-detecting-unexpected-table-volume-changes-with-acryl-observe",
+    readTime: "4",
+  },
+  {
+    title: "Detecting Unexpected Column Changes with Acryl Observe",
+    tag: "Blog",
+    backgroundImage:
+      "https://cdn.sanity.io/images/cqo9wkgf/production/6620e79f4221c7d750d4cfba4371dd52fcd13953-2000x1933.png?w=3840&q=75&fit=clip&auto=format",
+    link: "https://www.acryldata.io/blog/detecting-deep-data-quality-issues-with-column-level-assertions",
+    readTime: "5",
+  },
+  {
+    title: "Implementing Data Contracts with DataHub",
+    tag: "Blog",
+    backgroundImage:
+      "https://cdn.sanity.io/images/cqo9wkgf/production/af26da1550032891f46053fdf673a228ead0307e-2000x1121.png?w=3840&q=75&fit=clip&auto=format",
+    link: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management",
+    readTime: "4",
+  },
+];
+
+export default resourceData;
diff --git a/docs-website/src/pages/solutions/observability/_content/observeTestimonialsContent.js b/docs-website/src/pages/solutions/observability/_content/observeTestimonialsContent.js
new file mode 100644
index 0000000000000..cb87131502275
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/_content/observeTestimonialsContent.js
@@ -0,0 +1,10 @@
+const testimonialsData = {
+  title: "Enter end-to-end Data Observability.",
+  feature1: "Data Discovery",
+  feature2: "Governance",
+  feature1Link: "/solutions/discovery",
+  feature2Link: "/solutions/governance",
+  imgSrc: "/img/solutions/observe-icons-group.png",
+};
+
+export default testimonialsData;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/observability/_content/observeTilesContent.js b/docs-website/src/pages/solutions/observability/_content/observeTilesContent.js
new file mode 100644
index 0000000000000..e837c7122c905
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/_content/observeTilesContent.js
@@ -0,0 +1,77 @@
+const tilesContent = [
+  {
+      title: "Detect Data Quality Issues. <br />Effortlessly.",
+      theme: "dark",
+      tileItems: [
+        {
+          title: "Set it, then forget it.",
+          subtitle: "Create data quality checks effortlessly using the UI or API. Detect freshness, volume, and column value anomalies on tables as soon as they occur. ",
+          imgSrc: "/img/solutions/observe-tile-1.png",
+        },
+        {
+          title: "Cover your blind spots with AI-assisted anomaly detection.",
+          subtitle: "Turn on AI-powered data quality monitors in one click. Catch critical anomalies that would otherwise go undetected.",
+          imgSrc: "/img/solutions/observe-tile-2.png",
+        },
+        {
+          title: "Democratize data quality.",
+          subtitle: "Empower both non-technical and technical teams to create and monitor data quality checks—no code required.",
+          imgSrc: "/img/solutions/observe-tile-3.png",
+        },
+        {
+          title: "Built for your stack.",
+          subtitle: "Instantly start monitoring data on Snowflake, BigQuery, Redshift, Databricks, & more with out-of-the-box integrations.",
+          imgSrc: "/img/solutions/observe-tile-4.png",
+        }
+      ]
+    },
+    {
+      title: "Fix Data Fires.<br />Fast.",
+      theme: "light",
+      tileItems: [
+        {
+          title: "Get notified where you work.",
+          subtitle: "Find out when things go wrong, with alerts that reach your team where they work—whether it’s Slack, email, or anywhere else.",
+          imgSrc: "/img/solutions/observe-tile-5.png",
+        },
+        {
+          title: "Resolve incidents fast.",
+          subtitle: "Diagnose, debug and fix data issues with rich technical context including data statistics, downstream lineage, and detailed health history.",
+          imgSrc: "/img/solutions/observe-tile-6.png",
+        },
+        {
+          title: "Streamline data incident management.",
+          subtitle: "Centralize your process for tracking progress, sharing context, and keeping everyone who’s impacted in the loop.",
+          imgSrc: "/img/solutions/observe-tile-7.png",
+        }
+      ]
+    },
+    {
+      title: "Visualize the health of all your data.",
+      theme: "dark",
+      tileItems: [
+        {
+          title: "At-a-glance summary of your data quality.",
+          subtitle: "Understand key stats like percent of healthy tables, and table coverage. Easily drill down to see these metrics by domain, owners and more.",
+          imgSrc: "/img/solutions/observe-tile-8.png",
+        },
+        {
+          title: "Slice and dice data quality, your way.",
+          subtitle: "Get detailed breakdowns of assertion failures over the last N days by domains, tags, owners and more.",
+          imgSrc: "/img/solutions/observe-tile-9.png",
+        },
+        {
+          title: "Identify check failures over time.",
+          subtitle: "Easily spot spikes and recurring data quality failures with a customizable timeline view.",
+          imgSrc: "/img/solutions/observe-tile-10.png",
+        },
+        {
+          title: "Triage incidents and track progress.",
+          subtitle: "Understand the current state and historical patterns of data incidents, at a single glance. Take action fast.",
+          imgSrc: "/img/solutions/observe-tile-11.png",
+        }
+      ]
+    }
+  ];
+  
+export default tilesContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/observability/_content/observeTrialsContent.js b/docs-website/src/pages/solutions/observability/_content/observeTrialsContent.js
new file mode 100644
index 0000000000000..14eacd50b0faf
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/_content/observeTrialsContent.js
@@ -0,0 +1,24 @@
+const trialsContent = {
+      title: "Start building trust with your stakeholders, today.",
+      trialsCardItems: [
+        {
+          title: "Protect your mission-critical tables, reports, services, and more.",
+          imgSrc: "/img/solutions/trial-icon-lock.svg",
+          
+        },
+        {
+          title: "Know first, not last. Get notified where you work when things go wrong.",
+          imgSrc: "/img/solutions/trial-icon-alert.svg",
+        },
+        {
+          title: "Let AI detect the blindspots in your data quality checks.",
+          imgSrc: "/img/solutions/trial-icon-star.svg",
+        },
+        {
+          title: "Share documentation, compliance and health for any data asset with one link.",
+          imgSrc: "/img/solutions/trial-icon-link.svg",
+        }
+      ]
+    };
+  
+  export default trialsContent;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/observability/_content/observeUnifiedTabsContent.js b/docs-website/src/pages/solutions/observability/_content/observeUnifiedTabsContent.js
new file mode 100644
index 0000000000000..f02e60f06d1d6
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/_content/observeUnifiedTabsContent.js
@@ -0,0 +1,22 @@
+const unifiedTabsData = [
+  {
+    tabName: 'Detect',
+    title: 'Be the first to know when something goes wrong.',
+    description: 'Identify data quality issues and outages before they disrupt operations, damage trust and impact end users. Notify the right people where they work when things go wrong.',
+    image: '/img/solutions/unified-tab-detect.png',
+  },
+  {
+    tabName: 'Resolve',
+    title: 'Spend minutes, not days, resolving issues.',
+    description: 'Accelerate time-to-resolution with detailed lineage, documentation, and ownership information. Track incident status in one place and keep all stakeholders in the loop.',
+    image: '/img/solutions/unified-tab-resolve.png',
+  },
+  {
+    tabName: 'Unify',
+    title: 'One platform to rule them all.',
+    description: 'Empower your teams to discover healthy data on their own. Combine quality with lineage, documentation and ownership information to triage and resolve data quality issues quickly. Make data your competitive edge by unifying Data Quality, Discovery and Governance under one roof.',
+    image: '/img/solutions/unified-tab-unify.png',
+  },
+];
+  
+export default unifiedTabsData;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/observability/index.js b/docs-website/src/pages/solutions/observability/index.js
new file mode 100644
index 0000000000000..359bea914e5a1
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/index.js
@@ -0,0 +1,121 @@
+import React, { useState } from "react";
+import Layout from "@theme/Layout";
+import useDocusaurusContext from "@docusaurus/useDocusaurusContext";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+import Hero from "../_components/Hero";
+import Tiles from "../_components/Tiles";
+import Testimonials from "../_components/Testimonials";
+import CaseStudy from "../_components/CaseStudy";
+import QuickstartContent from "../_components/QuickstartContent";
+import styles from "./styles.module.scss";
+import CloseButton from "@ant-design/icons/CloseCircleFilled";
+import clsx from "clsx";
+import quickstartData from "./_content/observeQuickstartContent";
+import heroContent from "./_content/observeHeroContent";
+import caseStudyContent from "./_content/observeCaseStudyContent";
+import IntegrationsStatic from "../_components/IntegrationsStatic";
+import tilesContent from "./_content/observeTilesContent";
+import testimonialsData from "./_content/observeTestimonialsContent";
+import resourceData from "./_content/observeResourceContent";
+import UnifiedTabs from "../_components/UnifiedTabs";
+import unifiedTabsData from "./_content/observeUnifiedTabsContent";
+import trialsContent from "./_content/observeTrialsContent";
+import Trials from "../_components/Trials";
+import SlidingTabs from "../_components/SlidingTabs";
+
+function Home() {
+  const context = useDocusaurusContext();
+  const { siteConfig = {} } = context;
+
+  if (siteConfig.customFields.isSaas) {
+    window.location.replace("/docs");
+  }
+
+  const [isTourModalVisible, setIsTourModalVisible] = useState(false);
+  const onOpenTourModal = () => {
+    setIsTourModalVisible(true);
+  };
+  const onCloseTourModal = () => {
+    setIsTourModalVisible(false);
+  };
+  return !siteConfig.customFields.isSaas ? (
+    <Layout
+      title={siteConfig.tagline}
+      description="DataHub is a data discovery application built on an extensible data catalog that helps you tame the complexity of diverse data ecosystems."
+    >
+      {isTourModalVisible ? (
+        <div className="tourModal">
+          <div className="closeButtonWrapper" onClick={onCloseTourModal}>
+            <CloseButton />
+          </div>
+          <iframe src="https://www.acryldata.io/tour" />
+        </div>
+      ) : null}
+      <Hero onOpenTourModal={onOpenTourModal} heroContent={heroContent}/>
+      <IntegrationsStatic />
+      <QuickstartContent quickstartContent={quickstartData} />
+      <Testimonials testimonialsData={testimonialsData} />
+      <UnifiedTabs unifiedTabsData={unifiedTabsData}/>
+      <div className={clsx("testimonials", styles.testimonials)}>
+      <div className="testimonials__content">
+        <div className="testimonials__card">
+          <div className="testimonials__meta">
+            <div className="testimonials__logo">
+              <img src={useBaseUrl("/img/solutions/logo-depop.png")} />
+            </div>
+            <div className="testimonials__company">
+              <div className="testimonials__company_title">Depop</div>
+              <div className="testimonials__author_title">Olivier Tatard <br />Engineering Manager</div>
+            </div>
+            </div>
+          <div className="testimonials__text">
+            "We chose Acryl because we see the value of having both a data catalog and observability capabilities in one tool. Having data owners, maintainers, and consumers in one place streamlines incident management and allows for faster time to resolution."
+          </div>
+        </div>
+      </div>
+    </div>
+      <div>
+        {tilesContent.map((content, index) => (
+          <Tiles key={index} tilesContent={content} />
+        ))}
+      </div>
+      <CaseStudy caseStudyContent= {caseStudyContent}/>
+      <SlidingTabs />
+      <Trials onOpenTourModal={onOpenTourModal} trialsContent={trialsContent} />
+      <div className={styles.resource_container}>
+      <div className={styles.resource}>
+        <div className={styles.resource_heading}>
+          Resources
+        </div>
+        <div className={styles.card_row}>
+          <div className={styles.card_row_wrapper}>
+            {resourceData.map((resource) => (
+              <div className={styles.card} key={resource.link}>
+                <a className={styles.cardLink} href={resource.link}>
+                  {resource.tag ? <span className={styles.card_tag}>{resource.tag}</span> : null}
+                  <div class={styles.read_time}>{resource.readTime} mins read</div>
+                  <div className={styles.card_image} style={{ backgroundImage: `url(${resource.backgroundImage})` }}>
+                    <div className={styles.cardImageBackground} />
+                  </div>
+                  <div className={styles.card_content}>
+                    <div className={styles.card_heading}>
+                      {resource.title}
+                    </div>
+                    <div className={styles.read_more}>
+                      <a href={resource.link} className={styles.read_more}>
+                        Read more
+                      </a>
+                    </div>
+                  </div>
+                </a>
+              </div>
+            ))}
+          </div>
+        </div>
+      </div>
+    </div>
+    </Layout>
+  ) : null;
+}
+
+export default Home;
\ No newline at end of file
diff --git a/docs-website/src/pages/solutions/observability/styles.module.scss b/docs-website/src/pages/solutions/observability/styles.module.scss
new file mode 100644
index 0000000000000..2125e2237c9dd
--- /dev/null
+++ b/docs-website/src/pages/solutions/observability/styles.module.scss
@@ -0,0 +1,342 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  width: 80vw;
+  min-width: 900px;
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+.testimonials {
+  background: #f1f1f1;
+
+  :global {
+    .testimonials__content {
+      width: 80vw;
+      max-width: 1200px;
+      margin: 0 auto;
+      padding: 40px 0;
+
+      .testimonials__card {
+        margin: 2rem 4rem;
+        display: flex;
+
+        .testimonials__meta {
+          display: flex;
+        }
+
+        .testimonials__logo {
+          max-width: 120px;
+          margin: auto 1rem;
+        }
+        .testimonials__company {
+          width: 300px;
+          margin: auto 2rem;
+
+          .testimonials__company_title {
+            color: var(--primitives-text-text-heading, #373A47);
+            font-family: Manrope;
+            font-size: 36px;
+            font-style: normal;
+            font-weight: 700;
+            line-height: normal;
+            letter-spacing: -1.92px;
+            margin-bottom: 8px;
+
+          }
+         
+          .testimonials__author_title {
+            color: var(--primitives-text-text-heading, #373A47);
+            font-family: Manrope;
+            font-size: 20px;
+            font-style: normal;
+            font-weight: 300;
+            line-height: normal;
+            letter-spacing: -1.28px;
+          }
+        }
+
+        .testimonials__text {
+          width: 100%;
+          color: var(--primitives-text-text-heading, #373A47);
+          font-family: Manrope;
+          font-size: 20px;
+          font-style: normal;
+          font-weight: 400;
+          line-height: 140%;
+          margin: auto 2rem;
+        }
+      }
+    }
+
+    @media only screen and (max-width: 800px) {
+      .testimonials__content {
+        width: 100vw;
+        padding: 20px 0;
+
+        .testimonials__card {
+          flex-direction: column;
+          margin: 2rem 3rem;
+
+          .testimonials__meta {
+            display: flex;
+          }
+
+          .testimonials__logo {
+            max-width: 40px;
+            margin: auto 1rem;
+          }
+
+          .testimonials__company {
+            display: flex;
+            margin: auto 1rem;
+            flex-direction: column;
+
+            .testimonials__company_title {
+              font-size: 1.5rem;
+              margin-bottom: 0;
+            }
+            .testimonials__author_title {
+              padding-left: 0;
+            }
+          }
+
+          .testimonials__text {
+            width: 100%;
+            max-width: 100%;
+            margin: 2rem auto;
+          }
+        }
+      }
+      
+    }
+  }
+}
+
+
+// Responsiveness
+@media (max-width: 800px) {
+
+  .container {
+    flex-direction: column;
+    width: 90vw;
+    min-width: 0;
+    margin: 4rem auto;
+  }
+}
+
+
+.resource_container {
+  display: flex;
+  flex-direction: column;
+  background: #F4F4F5;
+  font-family: "Manrope";
+}
+.resource {
+  display: flex;
+  flex-direction: column;
+  width: 100vw;
+  margin: 5rem 0;
+
+  .resource_heading {
+    line-height: 4rem;
+    font-family: Manrope;
+    font-family: Manrope;
+    font-size: 3.25rem;
+    font-style: normal;
+    font-weight: 500;
+    padding-left: 5rem;
+    line-height: normal;
+    mix-blend-mode: luminosity;
+    width: 100%;
+    margin-bottom: 1rem;
+  }
+
+  .card_row::-webkit-scrollbar {
+    display: none;
+  }
+  .card_row {
+    margin-top: 8px;
+    overflow-x: scroll;
+    padding-right: 5rem;
+    padding-left: 5rem;
+    scrollbar-width: none;
+
+    .card_row_wrapper {
+      display: flex;
+      flex-direction: row;
+      align-items: center;
+      flex: 1;
+    }
+    .cardLink {
+      color: #000;
+      cursor: pointer;
+      display: block;
+      height: 100%;
+      
+      &:hover {
+        text-decoration: none;
+      }
+    }
+    
+    .card {
+      display: flex;
+      flex-direction: column;
+      justify-content: space-between;
+      flex-direction: column;
+      position: relative;
+      width: 300px;
+      height: 400px;
+      flex-shrink: 0;
+      border-radius: 12px;
+      background: white;
+      margin-right: 20px;
+      transition: box-shadow 0.2s ease-in-out;
+      box-shadow: 0px 2px 16px 0px rgba(55, 61, 68, 0.08);
+      overflow-y: visible;
+    
+    
+      .read_time {
+        position: absolute;
+        top: 16px;
+        right: 16px;
+        color: white;
+        font-size: 1rem;
+        z-index: 10;
+        text-shadow: 0px 1px 4px black;
+      }
+
+      .card_image {
+        position: relative;
+        height: 200px;
+        background-size: cover;
+        background-position: center;
+        border-radius: 12px 12px 0 0;
+        display: flex;
+        justify-content: center;
+        align-items: center;
+    
+        img {
+          max-width: 100%;
+          max-height: 100%;
+          object-fit: cover;
+        }
+      }
+    
+      .card_content {
+        padding: 16px;
+    
+        .card_heading {
+          font-size: 1.4rem;
+          line-height: 1.75rem;
+          font-weight: 600;
+          margin-bottom: 8px;
+        }
+
+        .read_more {
+          color: #1890ff;
+          font-weight: bold;
+          text-decoration: none;
+          font-size: 1rem;
+          position: absolute; 
+          bottom: 8px;
+          left: 8px;
+          width: 100%;
+          transition: all .2s;
+    
+          &:hover {
+            text-decoration: none;
+            opacity: .9;
+          }
+        }
+      }
+    
+      .card_tag {
+        position: absolute;
+        top: 16px;
+        left: 16px;
+        background-color: #F9F9F9;
+        color: black;
+        padding: 4px 24px;
+        border-radius: 20px;
+        font-size: 1.2rem;
+        font-weight: bold;
+        z-index: 10;
+      }
+    }
+    
+  }
+
+  .bottom_line {
+    cursor: pointer;
+    text-decoration: none;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    color: #12b0fb;
+    font-size: 1.1rem;
+    font-weight: 500;
+    margin-top: 50px;
+    margin-bottom: 10px;
+
+    span {
+      line-height: 10px;
+      font-size: 1.5rem;
+      margin-left: 10px;
+    }
+  }
+}
+
+@media (max-width: 800px) {
+  .resource {
+    .resource_heading {
+      padding-left: 2rem;
+      font-size: 2rem;
+    }
+
+    .card_row {
+      padding-left: 0;
+
+      .card_row_wrapper {
+        padding: 0 2rem;
+      }
+    }
+  }
+  .case_study {
+    .case_study_heading {
+      text-align: center;
+      font-family: "Manrope";
+
+      div {
+        width: 80%;
+        margin: auto;
+        font-size: 2rem;
+        line-height: normal;
+        font-weight: 400;
+      }
+      p {
+        width: 80%;
+        margin: auto;
+        font-size: 1.1rem;
+        line-height: 1.5rem;
+        margin-top: 12px;
+      }
+    }
+
+    .bottom_line {
+      cursor: pointer;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      color: #12b0fb;
+      font-size: 1.1rem;
+      font-weight: 500;
+      margin-top: 50px;
+      margin-bottom: 10px;
+
+      span {
+        font-size: 1.5rem;
+      }
+    }
+  }
+}
diff --git a/docs-website/src/styles/global.scss b/docs-website/src/styles/global.scss
index e256c752b4a0b..31261ea3d940d 100644
--- a/docs-website/src/styles/global.scss
+++ b/docs-website/src/styles/global.scss
@@ -31,7 +31,7 @@
   --ifm-navbar-item-padding-horizontal: 1rem;
 
   /* Announcement Bar */
-  --docusaurus-announcement-bar-height: 60px !important;
+  --docusaurus-announcement-bar-height: 48px !important;
 
   /* Rule */
   --ifm-hr-border-width: 1px 0 0 0;
@@ -116,21 +116,34 @@ div[class^="announcementBar"] {
     >div {
       display: flex;
       align-items: center;
+      > div {
+        @media (max-width: 580px) {
+            display: none;
+          }
+      }
+      a>span {
+        @media (max-width: 580px) {
+          display: none;
+        }
+      }
 
       >p {
         text-align: left;
         line-height: 1.1rem;
         margin: 0;
 
-        >span {
-          @media (max-width: 780px) {
-            display: none;
-          }
-        }
-
-        @media (max-width: 480px) {
-          display: none;
+        @media (max-width: 580px) {
+          font-size: .9rem;
         }
+        // >span {
+        //   @media (max-width: 780px) {
+        //     display: none;
+        //   }
+        // }
+
+        // @media (max-width: 480px) {
+        //   display: none;
+        // }
       }
     }
 
@@ -141,8 +154,9 @@ div[class^="announcementBar"] {
     }
 
     a {
-      color: var(--ifm-button-color);
+      color: #EFB300;
       text-decoration: none;
+      font-size: 1rem
     }
   }
 }
diff --git a/docs-website/src/theme/Footer/index.js b/docs-website/src/theme/Footer/index.js
index 74e6eb7d09a7b..fe0086993ee9e 100644
--- a/docs-website/src/theme/Footer/index.js
+++ b/docs-website/src/theme/Footer/index.js
@@ -1,11 +1,15 @@
 import React from "react";
+import { useLocation } from "react-router-dom"; // Import useLocation from react-router-dom
 import Footer from "@theme-original/Footer";
 import MarkpromptHelp from "../../components/MarkpromptHelp";
 
 export default function FooterWrapper(props) {
+  const location = useLocation(); // Get the current location
+  const isDocsPage = location.pathname.startsWith("/docs"); // Check if the path starts with /docs
+
   return (
     <>
-      <MarkpromptHelp />
+      {isDocsPage && <MarkpromptHelp />}
       <Footer {...props} />
     </>
   );
diff --git a/docs-website/static/img/adoption-stories/adoption-stories-apple.png b/docs-website/static/img/adoption-stories/adoption-stories-apple.png
new file mode 100644
index 0000000000000..7ca5b06300abc
Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-apple.png differ
diff --git a/docs-website/static/img/logos/companies/apple_text.png b/docs-website/static/img/logos/companies/apple_text.png
new file mode 100644
index 0000000000000..59ede054a96db
Binary files /dev/null and b/docs-website/static/img/logos/companies/apple_text.png differ
diff --git a/docs-website/static/img/logos/platforms/hudi.png b/docs-website/static/img/logos/platforms/hudi.png
new file mode 100644
index 0000000000000..c5e79bcc86ce3
Binary files /dev/null and b/docs-website/static/img/logos/platforms/hudi.png differ
diff --git a/docs-website/static/img/solutions/case-study-card-bg.png b/docs-website/static/img/solutions/case-study-card-bg.png
new file mode 100644
index 0000000000000..7377c7282842e
Binary files /dev/null and b/docs-website/static/img/solutions/case-study-card-bg.png differ
diff --git a/docs-website/static/img/solutions/checkout.jpg b/docs-website/static/img/solutions/checkout.jpg
new file mode 100644
index 0000000000000..4a766475cc599
Binary files /dev/null and b/docs-website/static/img/solutions/checkout.jpg differ
diff --git a/docs-website/static/img/solutions/communities.png b/docs-website/static/img/solutions/communities.png
new file mode 100644
index 0000000000000..f15a7c1636191
Binary files /dev/null and b/docs-website/static/img/solutions/communities.png differ
diff --git a/docs-website/static/img/solutions/discovery-icons-group.png b/docs-website/static/img/solutions/discovery-icons-group.png
new file mode 100644
index 0000000000000..a86094a76409c
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-icons-group.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-1.png b/docs-website/static/img/solutions/discovery-tile-1.png
new file mode 100644
index 0000000000000..185cd8dd48353
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-1.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-2.png b/docs-website/static/img/solutions/discovery-tile-2.png
new file mode 100644
index 0000000000000..185838f3086ff
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-2.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-3.png b/docs-website/static/img/solutions/discovery-tile-3.png
new file mode 100644
index 0000000000000..63c44271a2eaf
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-3.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-4.png b/docs-website/static/img/solutions/discovery-tile-4.png
new file mode 100644
index 0000000000000..f9c9af7096492
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-4.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-5.png b/docs-website/static/img/solutions/discovery-tile-5.png
new file mode 100644
index 0000000000000..04a78d27ae898
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-5.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-6.png b/docs-website/static/img/solutions/discovery-tile-6.png
new file mode 100644
index 0000000000000..2a347ebc0de53
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-6.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-7.png b/docs-website/static/img/solutions/discovery-tile-7.png
new file mode 100644
index 0000000000000..a1af7d8f4dc38
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-7.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-8.png b/docs-website/static/img/solutions/discovery-tile-8.png
new file mode 100644
index 0000000000000..c10fa4e690e47
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-8.png differ
diff --git a/docs-website/static/img/solutions/discovery-tile-9.png b/docs-website/static/img/solutions/discovery-tile-9.png
new file mode 100644
index 0000000000000..a3e93d6fc7025
Binary files /dev/null and b/docs-website/static/img/solutions/discovery-tile-9.png differ
diff --git a/docs-website/static/img/solutions/governance-icons-group.png b/docs-website/static/img/solutions/governance-icons-group.png
new file mode 100644
index 0000000000000..464b6bed20563
Binary files /dev/null and b/docs-website/static/img/solutions/governance-icons-group.png differ
diff --git a/docs-website/static/img/solutions/governance-tile-1.png b/docs-website/static/img/solutions/governance-tile-1.png
new file mode 100644
index 0000000000000..28ba604ff6b20
Binary files /dev/null and b/docs-website/static/img/solutions/governance-tile-1.png differ
diff --git a/docs-website/static/img/solutions/governance-tile-2.png b/docs-website/static/img/solutions/governance-tile-2.png
new file mode 100644
index 0000000000000..ae28d833808a2
Binary files /dev/null and b/docs-website/static/img/solutions/governance-tile-2.png differ
diff --git a/docs-website/static/img/solutions/governance-tile-3.png b/docs-website/static/img/solutions/governance-tile-3.png
new file mode 100644
index 0000000000000..534c901e88821
Binary files /dev/null and b/docs-website/static/img/solutions/governance-tile-3.png differ
diff --git a/docs-website/static/img/solutions/governance-tile-4.png b/docs-website/static/img/solutions/governance-tile-4.png
new file mode 100644
index 0000000000000..5527df93309db
Binary files /dev/null and b/docs-website/static/img/solutions/governance-tile-4.png differ
diff --git a/docs-website/static/img/solutions/governance-tile-5.png b/docs-website/static/img/solutions/governance-tile-5.png
new file mode 100644
index 0000000000000..73cbbcaf72c6a
Binary files /dev/null and b/docs-website/static/img/solutions/governance-tile-5.png differ
diff --git a/docs-website/static/img/solutions/governance-tile-6.png b/docs-website/static/img/solutions/governance-tile-6.png
new file mode 100644
index 0000000000000..5eeabc84f72f4
Binary files /dev/null and b/docs-website/static/img/solutions/governance-tile-6.png differ
diff --git a/docs-website/static/img/solutions/governance-tile-7.png b/docs-website/static/img/solutions/governance-tile-7.png
new file mode 100644
index 0000000000000..0914077227800
Binary files /dev/null and b/docs-website/static/img/solutions/governance-tile-7.png differ
diff --git a/docs-website/static/img/solutions/hero-background.png b/docs-website/static/img/solutions/hero-background.png
new file mode 100644
index 0000000000000..3daeaca68dc56
Binary files /dev/null and b/docs-website/static/img/solutions/hero-background.png differ
diff --git a/docs-website/static/img/solutions/hero-discovery.png b/docs-website/static/img/solutions/hero-discovery.png
new file mode 100644
index 0000000000000..28f962d4b84a0
Binary files /dev/null and b/docs-website/static/img/solutions/hero-discovery.png differ
diff --git a/docs-website/static/img/solutions/hero-governance.png b/docs-website/static/img/solutions/hero-governance.png
new file mode 100644
index 0000000000000..e94c1d1303e9f
Binary files /dev/null and b/docs-website/static/img/solutions/hero-governance.png differ
diff --git a/docs-website/static/img/solutions/hero-observe.png b/docs-website/static/img/solutions/hero-observe.png
new file mode 100644
index 0000000000000..91ed48d5f45ef
Binary files /dev/null and b/docs-website/static/img/solutions/hero-observe.png differ
diff --git a/docs-website/static/img/solutions/icon-calendar.png b/docs-website/static/img/solutions/icon-calendar.png
new file mode 100644
index 0000000000000..56a2fda29f25f
Binary files /dev/null and b/docs-website/static/img/solutions/icon-calendar.png differ
diff --git a/docs-website/static/img/solutions/icon-cloud.png b/docs-website/static/img/solutions/icon-cloud.png
new file mode 100644
index 0000000000000..fccb4543fac28
Binary files /dev/null and b/docs-website/static/img/solutions/icon-cloud.png differ
diff --git a/docs-website/static/img/solutions/icon-metric.png b/docs-website/static/img/solutions/icon-metric.png
new file mode 100644
index 0000000000000..ad3d725bef6e9
Binary files /dev/null and b/docs-website/static/img/solutions/icon-metric.png differ
diff --git a/docs-website/static/img/solutions/icon-migration.png b/docs-website/static/img/solutions/icon-migration.png
new file mode 100644
index 0000000000000..c4fadac2150ba
Binary files /dev/null and b/docs-website/static/img/solutions/icon-migration.png differ
diff --git a/docs-website/static/img/solutions/icon-revenue.png b/docs-website/static/img/solutions/icon-revenue.png
new file mode 100644
index 0000000000000..6af1f18e0aeaa
Binary files /dev/null and b/docs-website/static/img/solutions/icon-revenue.png differ
diff --git a/docs-website/static/img/solutions/icon-wrench.png b/docs-website/static/img/solutions/icon-wrench.png
new file mode 100644
index 0000000000000..cb050b3f7239e
Binary files /dev/null and b/docs-website/static/img/solutions/icon-wrench.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-1.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-1.png
new file mode 100644
index 0000000000000..67a441252fa3f
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-1.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-2.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-2.png
new file mode 100644
index 0000000000000..091619f03300f
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-2.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-3.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-3.png
new file mode 100644
index 0000000000000..738e251c327b8
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-3.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-4.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-4.png
new file mode 100644
index 0000000000000..92d89a262adde
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-4.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-5.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-5.png
new file mode 100644
index 0000000000000..c4d69b1ed3bc1
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-5.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-6.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-6.png
new file mode 100644
index 0000000000000..c22402e99d1f3
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-6.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-7.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-7.png
new file mode 100644
index 0000000000000..acc17cb75d585
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-7.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-8.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-8.png
new file mode 100644
index 0000000000000..d9bb08766f527
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-8.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-1.png b/docs-website/static/img/solutions/integrations/logo-integration-1.png
new file mode 100644
index 0000000000000..091619f03300f
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-1.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-10.png b/docs-website/static/img/solutions/integrations/logo-integration-10.png
new file mode 100644
index 0000000000000..acc17cb75d585
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-10.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-11.png b/docs-website/static/img/solutions/integrations/logo-integration-11.png
new file mode 100644
index 0000000000000..d9bb08766f527
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-11.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-2.png b/docs-website/static/img/solutions/integrations/logo-integration-2.png
new file mode 100644
index 0000000000000..738e251c327b8
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-2.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-3.png b/docs-website/static/img/solutions/integrations/logo-integration-3.png
new file mode 100644
index 0000000000000..92d89a262adde
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-3.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-4.png b/docs-website/static/img/solutions/integrations/logo-integration-4.png
new file mode 100644
index 0000000000000..67a441252fa3f
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-4.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-5.png b/docs-website/static/img/solutions/integrations/logo-integration-5.png
new file mode 100644
index 0000000000000..acc17cb75d585
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-5.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-6.png b/docs-website/static/img/solutions/integrations/logo-integration-6.png
new file mode 100644
index 0000000000000..d9bb08766f527
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-6.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-7.png b/docs-website/static/img/solutions/integrations/logo-integration-7.png
new file mode 100644
index 0000000000000..2672e9c1c2a8e
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-7.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-8.png b/docs-website/static/img/solutions/integrations/logo-integration-8.png
new file mode 100644
index 0000000000000..4b93e23846004
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-8.png differ
diff --git a/docs-website/static/img/solutions/integrations/logo-integration-9.png b/docs-website/static/img/solutions/integrations/logo-integration-9.png
new file mode 100644
index 0000000000000..eee40d590a148
Binary files /dev/null and b/docs-website/static/img/solutions/integrations/logo-integration-9.png differ
diff --git a/docs-website/static/img/solutions/lock.png b/docs-website/static/img/solutions/lock.png
new file mode 100644
index 0000000000000..ce7cda258c4ce
Binary files /dev/null and b/docs-website/static/img/solutions/lock.png differ
diff --git a/docs-website/static/img/solutions/logo-depop.png b/docs-website/static/img/solutions/logo-depop.png
new file mode 100644
index 0000000000000..337d0087be6fa
Binary files /dev/null and b/docs-website/static/img/solutions/logo-depop.png differ
diff --git a/docs-website/static/img/solutions/logo-dpg-media.png b/docs-website/static/img/solutions/logo-dpg-media.png
new file mode 100644
index 0000000000000..0252a018d65b7
Binary files /dev/null and b/docs-website/static/img/solutions/logo-dpg-media.png differ
diff --git a/docs-website/static/img/solutions/logo-myob.png b/docs-website/static/img/solutions/logo-myob.png
new file mode 100644
index 0000000000000..a166565a7ed85
Binary files /dev/null and b/docs-website/static/img/solutions/logo-myob.png differ
diff --git a/docs-website/static/img/solutions/logo-notion.png b/docs-website/static/img/solutions/logo-notion.png
new file mode 100644
index 0000000000000..ea0601609972d
Binary files /dev/null and b/docs-website/static/img/solutions/logo-notion.png differ
diff --git a/docs-website/static/img/solutions/miro.png b/docs-website/static/img/solutions/miro.png
new file mode 100644
index 0000000000000..3009fcbe03a29
Binary files /dev/null and b/docs-website/static/img/solutions/miro.png differ
diff --git a/docs-website/static/img/solutions/observe-icons-group.png b/docs-website/static/img/solutions/observe-icons-group.png
new file mode 100644
index 0000000000000..704b4a009f899
Binary files /dev/null and b/docs-website/static/img/solutions/observe-icons-group.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-1.png b/docs-website/static/img/solutions/observe-tile-1.png
new file mode 100644
index 0000000000000..61dce01ff4a41
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-1.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-10.png b/docs-website/static/img/solutions/observe-tile-10.png
new file mode 100644
index 0000000000000..92bcb393c3f1d
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-10.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-11.png b/docs-website/static/img/solutions/observe-tile-11.png
new file mode 100644
index 0000000000000..a4c3d126ef4bd
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-11.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-2.png b/docs-website/static/img/solutions/observe-tile-2.png
new file mode 100644
index 0000000000000..de5786a925aa1
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-2.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-3.png b/docs-website/static/img/solutions/observe-tile-3.png
new file mode 100644
index 0000000000000..74ad1eb976950
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-3.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-4.png b/docs-website/static/img/solutions/observe-tile-4.png
new file mode 100644
index 0000000000000..0aca86f15a791
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-4.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-5.png b/docs-website/static/img/solutions/observe-tile-5.png
new file mode 100644
index 0000000000000..2f4839cc65f7f
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-5.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-6.png b/docs-website/static/img/solutions/observe-tile-6.png
new file mode 100644
index 0000000000000..ba97412c0576c
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-6.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-7.png b/docs-website/static/img/solutions/observe-tile-7.png
new file mode 100644
index 0000000000000..2b882d54857ba
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-7.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-8.png b/docs-website/static/img/solutions/observe-tile-8.png
new file mode 100644
index 0000000000000..51ace6a819457
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-8.png differ
diff --git a/docs-website/static/img/solutions/observe-tile-9.png b/docs-website/static/img/solutions/observe-tile-9.png
new file mode 100644
index 0000000000000..7c40b3a665522
Binary files /dev/null and b/docs-website/static/img/solutions/observe-tile-9.png differ
diff --git a/docs-website/static/img/solutions/optum.jpg b/docs-website/static/img/solutions/optum.jpg
new file mode 100644
index 0000000000000..8be6a260b0129
Binary files /dev/null and b/docs-website/static/img/solutions/optum.jpg differ
diff --git a/docs-website/static/img/solutions/persona_compliance_officers.png b/docs-website/static/img/solutions/persona_compliance_officers.png
new file mode 100644
index 0000000000000..07780a7c6c41b
Binary files /dev/null and b/docs-website/static/img/solutions/persona_compliance_officers.png differ
diff --git a/docs-website/static/img/solutions/persona_developers.png b/docs-website/static/img/solutions/persona_developers.png
new file mode 100644
index 0000000000000..8d71b4697f3d6
Binary files /dev/null and b/docs-website/static/img/solutions/persona_developers.png differ
diff --git a/docs-website/static/img/solutions/persona_owners_and_smes.png b/docs-website/static/img/solutions/persona_owners_and_smes.png
new file mode 100644
index 0000000000000..101f019c11485
Binary files /dev/null and b/docs-website/static/img/solutions/persona_owners_and_smes.png differ
diff --git a/docs-website/static/img/solutions/personas-mobile.png b/docs-website/static/img/solutions/personas-mobile.png
new file mode 100644
index 0000000000000..ca64e27a4c730
Binary files /dev/null and b/docs-website/static/img/solutions/personas-mobile.png differ
diff --git a/docs-website/static/img/solutions/rocket-launch.png b/docs-website/static/img/solutions/rocket-launch.png
new file mode 100644
index 0000000000000..1da699504cab8
Binary files /dev/null and b/docs-website/static/img/solutions/rocket-launch.png differ
diff --git a/docs-website/static/img/solutions/sliding-tab-bg.png b/docs-website/static/img/solutions/sliding-tab-bg.png
new file mode 100644
index 0000000000000..ac02e994dd109
Binary files /dev/null and b/docs-website/static/img/solutions/sliding-tab-bg.png differ
diff --git a/docs-website/static/img/solutions/trial-icon-alert.svg b/docs-website/static/img/solutions/trial-icon-alert.svg
new file mode 100644
index 0000000000000..407a4322ad1a0
--- /dev/null
+++ b/docs-website/static/img/solutions/trial-icon-alert.svg
@@ -0,0 +1,6 @@
+<svg width="86" height="87" viewBox="0 0 86 87" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle opacity="0.3" cx="43" cy="43.7266" r="42.5" fill="#EFFBFF" stroke="#838BA8"/>
+<circle opacity="0.6" cx="43" cy="43.7265" r="34.041" fill="#838BA8" stroke="#838BA8"/>
+<circle cx="42.6475" cy="43.374" r="25.7295" fill="#838BA8"/>
+<path d="M51.9278 38.4072C51.7512 38.4986 51.5456 38.5162 51.3561 38.4561C51.1666 38.396 51.0087 38.2631 50.9171 38.0866C50.2013 36.6724 49.1177 35.4771 47.7803 34.6263C47.6969 34.5737 47.6247 34.5052 47.5678 34.4248C47.511 34.3443 47.4705 34.2534 47.4487 34.1573C47.4269 34.0612 47.4243 33.9618 47.441 33.8646C47.4577 33.7675 47.4933 33.6746 47.5459 33.5913C47.5984 33.5079 47.6669 33.4358 47.7474 33.3789C47.8278 33.322 47.9187 33.2815 48.0148 33.2597C48.1109 33.238 48.2104 33.2354 48.3075 33.252C48.4046 33.2687 48.4975 33.3043 48.5809 33.3569C50.1396 34.3549 51.4052 35.749 52.2484 37.3966C52.3398 37.5731 52.3574 37.7788 52.2973 37.9683C52.2371 38.1578 52.1042 38.3157 51.9278 38.4072ZM34.2756 38.4916C34.413 38.4915 34.5477 38.4537 34.6651 38.3823C34.7825 38.3109 34.878 38.2086 34.9412 38.0866C35.6571 36.6724 36.7406 35.4771 38.0781 34.6263C38.2464 34.5201 38.3657 34.3514 38.4096 34.1573C38.4536 33.9632 38.4186 33.7596 38.3124 33.5913C38.2063 33.423 38.0376 33.3037 37.8435 33.2597C37.6494 33.2158 37.4458 33.2507 37.2774 33.3569C35.7188 34.3549 34.4531 35.749 33.6099 37.3966C33.5507 37.5109 33.5219 37.6385 33.5263 37.7672C33.5307 37.8958 33.5681 38.0212 33.635 38.1312C33.7019 38.2412 33.7959 38.3321 33.9081 38.3952C34.0203 38.4584 34.1468 38.4915 34.2756 38.4916ZM51.7224 48.236C51.8553 48.4638 51.9258 48.7227 51.9267 48.9865C51.9276 49.2502 51.8589 49.5096 51.7276 49.7383C51.5963 49.9671 51.407 50.1572 51.1788 50.2894C50.9506 50.4217 50.6915 50.4914 50.4278 50.4916H46.6028C46.4306 51.3393 45.9707 52.1015 45.3009 52.6489C44.6312 53.1963 43.7928 53.4954 42.9278 53.4954C42.0627 53.4954 41.2243 53.1963 40.5546 52.6489C39.8848 52.1015 39.4249 51.3393 39.2528 50.4916H35.4278C35.1641 50.4911 34.9053 50.4211 34.6774 50.2887C34.4494 50.1563 34.2604 49.9662 34.1293 49.7375C33.9982 49.5088 33.9297 49.2496 33.9307 48.986C33.9317 48.7224 34.0021 48.4637 34.1349 48.236C34.9796 46.7782 35.4278 44.7053 35.4278 42.2416C35.4278 40.2525 36.2179 38.3448 37.6245 36.9383C39.031 35.5318 40.9386 34.7416 42.9278 34.7416C44.9169 34.7416 46.8245 35.5318 48.2311 36.9383C49.6376 38.3448 50.4278 40.2525 50.4278 42.2416C50.4278 44.7044 50.8759 46.7772 51.7224 48.236ZM45.0484 50.4916H40.8071C40.9625 50.9298 41.2498 51.3091 41.6296 51.5774C42.0093 51.8456 42.4628 51.9896 42.9278 51.9896C43.3927 51.9896 43.8462 51.8456 44.2259 51.5774C44.6057 51.3091 44.893 50.9298 45.0484 50.4916Z" fill="white"/>
+</svg>
diff --git a/docs-website/static/img/solutions/trial-icon-language.png b/docs-website/static/img/solutions/trial-icon-language.png
new file mode 100644
index 0000000000000..7c9ee14e1e582
Binary files /dev/null and b/docs-website/static/img/solutions/trial-icon-language.png differ
diff --git a/docs-website/static/img/solutions/trial-icon-lightening.png b/docs-website/static/img/solutions/trial-icon-lightening.png
new file mode 100644
index 0000000000000..8ea71c819f8a1
Binary files /dev/null and b/docs-website/static/img/solutions/trial-icon-lightening.png differ
diff --git a/docs-website/static/img/solutions/trial-icon-link.svg b/docs-website/static/img/solutions/trial-icon-link.svg
new file mode 100644
index 0000000000000..b6fa92c9f2534
--- /dev/null
+++ b/docs-website/static/img/solutions/trial-icon-link.svg
@@ -0,0 +1,6 @@
+<svg width="86" height="87" viewBox="0 0 86 87" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle opacity="0.3" cx="43" cy="43.1084" r="42.5" fill="#838BA8" stroke="#838BA8"/>
+<circle opacity="0.6" cx="43" cy="43.1084" r="34.041" fill="#838BA8" stroke="#838BA8"/>
+<circle cx="42.6475" cy="42.7559" r="25.7295" fill="#838BA8"/>
+<path d="M51.8438 39.6479C51.422 39.6457 51.0037 39.7253 50.6122 39.8823L47.0391 35.4221C47.3328 34.9991 47.5228 34.5129 47.5938 34.0028C47.6647 33.4927 47.6146 32.9731 47.4475 32.486C47.2804 31.9989 47.001 31.5579 46.6318 31.1988C46.2627 30.8397 45.8142 30.5726 45.3227 30.4189C44.8311 30.2653 44.3103 30.2295 43.8024 30.3145C43.2945 30.3995 42.8137 30.6028 42.3989 30.9081C41.9842 31.2133 41.6471 31.6119 41.4149 32.0716C41.1827 32.5313 41.062 33.0392 41.0626 33.5541C41.0625 33.6692 41.0684 33.7842 41.0802 33.8987L35.6344 36.1674C35.3064 35.7651 34.8876 35.4463 34.4124 35.2373C33.9372 35.0283 33.4192 34.935 32.9009 34.9651C32.3826 34.9952 31.8789 35.1479 31.4311 35.4106C30.9833 35.6732 30.6043 36.0384 30.3251 36.4761C30.0458 36.9137 29.8745 37.4114 29.825 37.9282C29.7755 38.4449 29.8494 38.9661 30.0405 39.4488C30.2316 39.9314 30.5345 40.3619 30.9243 40.7047C31.3141 41.0476 31.7798 41.293 32.2829 41.4209L32.4997 46.8561C32.0029 47.0257 31.5541 47.3121 31.1909 47.6911C30.8277 48.0701 30.5607 48.5307 30.4124 49.0342C30.2641 49.5378 30.2387 50.0696 30.3385 50.585C30.4383 51.1003 30.6602 51.5842 30.9858 51.9961C31.3113 52.4079 31.7309 52.7356 32.2093 52.9517C32.6877 53.1678 33.2109 53.2659 33.7351 53.2379C34.2593 53.2099 34.7691 53.0565 35.2218 52.7906C35.6744 52.5248 36.0566 52.1542 36.3364 51.71L42.0001 53.2967C42.008 53.8245 42.1432 54.3425 42.3941 54.8069C42.6449 55.2713 43.0041 55.6683 43.4412 55.9642C43.8782 56.2602 44.3802 56.4463 44.9046 56.5069C45.4289 56.5674 45.9601 56.5006 46.4531 56.312C46.9461 56.1235 47.3863 55.8188 47.7364 55.4238C48.0865 55.0288 48.3362 54.5552 48.4642 54.0432C48.5922 53.5311 48.5949 52.9957 48.4718 52.4824C48.3488 51.9692 48.1037 51.4931 47.7575 51.0948L50.9462 46.085C51.3975 46.2135 51.8711 46.2437 52.3351 46.1735C52.7991 46.1034 53.2426 45.9345 53.6358 45.6784C54.0289 45.4222 54.3626 45.0847 54.6142 44.6886C54.8658 44.2925 55.0296 43.8471 55.0944 43.3823C55.1592 42.9176 55.1236 42.4443 54.9899 41.9945C54.8562 41.5447 54.6277 41.1288 54.3196 40.7748C54.0115 40.4209 53.6311 40.1371 53.204 39.9427C52.7769 39.7483 52.3131 39.6478 51.8438 39.6479ZM35.3427 40.6276L40.3266 43.5151L35.468 47.2909C35.1379 47.0547 34.7667 46.8819 34.3735 46.7811L34.1567 41.3459C34.5987 41.1937 35.003 40.9488 35.3427 40.6276ZM44.5278 36.8354C44.8876 36.8157 45.2416 36.7365 45.5755 36.601L49.1485 41.0612C48.9206 41.3884 48.7546 41.7546 48.6587 42.1416L43.2012 42.4018L44.5278 36.8354ZM36.62 48.7709L41.5313 44.9518L43.3993 50.5569C43.0436 50.8077 42.7409 51.1263 42.5087 51.4944L36.8438 49.9053C36.837 49.5169 36.7612 49.1328 36.62 48.7709ZM45.2813 49.9604H45.177L43.2809 44.2733L48.7477 44.0131C48.8859 44.4004 49.0958 44.7582 49.3665 45.0678L46.1778 50.0776C45.8858 49.9973 45.5841 49.9579 45.2813 49.9604ZM41.8032 35.6284C42.0551 35.9354 42.3599 36.195 42.7032 36.3948L41.379 41.9553L36.2848 39.0057C36.3448 38.7554 36.3751 38.499 36.3751 38.2416C36.3751 38.1266 36.3693 38.0116 36.3575 37.8971L41.8032 35.6284Z" fill="white"/>
+</svg>
diff --git a/docs-website/static/img/solutions/trial-icon-lock.svg b/docs-website/static/img/solutions/trial-icon-lock.svg
new file mode 100644
index 0000000000000..cfbbe5b8668e0
--- /dev/null
+++ b/docs-website/static/img/solutions/trial-icon-lock.svg
@@ -0,0 +1,6 @@
+<svg width="86" height="87" viewBox="0 0 86 87" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle opacity="0.3" cx="43" cy="43.6084" r="42.5" fill="#838BA8" stroke="#838BA8"/>
+<circle opacity="0.6" cx="43" cy="43.6084" r="34.041" fill="#838BA8" stroke="#838BA8"/>
+<circle cx="42.6475" cy="43.2559" r="25.7295" fill="#838BA8"/>
+<path d="M51.8052 37.9778H48.0542V35.1646C48.0542 33.6723 47.4614 32.2412 46.4062 31.186C45.3511 30.1309 43.92 29.5381 42.4277 29.5381C40.9355 29.5381 39.5044 30.1309 38.4492 31.186C37.3941 32.2412 36.8013 33.6723 36.8013 35.1646V37.9778H33.0503C32.5529 37.9778 32.0758 38.1754 31.7241 38.5271C31.3724 38.8788 31.1748 39.3559 31.1748 39.8533V52.9817C31.1748 53.4791 31.3724 53.9561 31.7241 54.3079C32.0758 54.6596 32.5529 54.8572 33.0503 54.8572H51.8052C52.3026 54.8572 52.7796 54.6596 53.1313 54.3079C53.4831 53.9561 53.6807 53.4791 53.6807 52.9817V39.8533C53.6807 39.3559 53.4831 38.8788 53.1313 38.5271C52.7796 38.1754 52.3026 37.9778 51.8052 37.9778ZM43.3655 47.1946V50.1685C43.3655 50.4172 43.2667 50.6557 43.0908 50.8315C42.915 51.0074 42.6764 51.1062 42.4277 51.1062C42.179 51.1062 41.9405 51.0074 41.7646 50.8315C41.5888 50.6557 41.49 50.4172 41.49 50.1685V47.1946C40.8643 46.9734 40.337 46.5382 40.0012 45.9657C39.6654 45.3933 39.5428 44.7206 39.655 44.0666C39.7672 43.4125 40.1071 42.8192 40.6145 42.3914C41.1218 41.9637 41.7641 41.7291 42.4277 41.7291C43.0914 41.7291 43.7336 41.9637 44.241 42.3914C44.7484 42.8192 45.0882 43.4125 45.2005 44.0666C45.3127 44.7206 45.1901 45.3933 44.8543 45.9657C44.5185 46.5382 43.9912 46.9734 43.3655 47.1946ZM46.1787 37.9778H38.6768V35.1646C38.6768 34.1697 39.0719 33.2157 39.7754 32.5122C40.4788 31.8088 41.4329 31.4136 42.4277 31.4136C43.4226 31.4136 44.3766 31.8088 45.0801 32.5122C45.7835 33.2157 46.1787 34.1697 46.1787 35.1646V37.9778Z" fill="white"/>
+</svg>
diff --git a/docs-website/static/img/solutions/trial-icon-owner.png b/docs-website/static/img/solutions/trial-icon-owner.png
new file mode 100644
index 0000000000000..b5382acfe5aba
Binary files /dev/null and b/docs-website/static/img/solutions/trial-icon-owner.png differ
diff --git a/docs-website/static/img/solutions/trial-icon-standard.png b/docs-website/static/img/solutions/trial-icon-standard.png
new file mode 100644
index 0000000000000..e1452a4b33d0b
Binary files /dev/null and b/docs-website/static/img/solutions/trial-icon-standard.png differ
diff --git a/docs-website/static/img/solutions/trial-icon-star.svg b/docs-website/static/img/solutions/trial-icon-star.svg
new file mode 100644
index 0000000000000..527a7fb92e668
--- /dev/null
+++ b/docs-website/static/img/solutions/trial-icon-star.svg
@@ -0,0 +1,6 @@
+<svg width="86" height="87" viewBox="0 0 86 87" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle opacity="0.2" cx="43" cy="43.0156" r="42.5" fill="#838BA8" stroke="#838BA8"/>
+<circle opacity="0.5" cx="43" cy="43.0156" r="34.041" fill="#838BA8" stroke="#838BA8"/>
+<circle cx="42.6475" cy="42.6631" r="25.2295" fill="#838BA8" stroke="#838BA8"/>
+<path d="M50.4999 44.0879C50.5018 44.3937 50.4089 44.6926 50.234 44.9434C50.059 45.1942 49.8107 45.3847 49.523 45.4885L44.6874 47.2754L42.9062 52.1148C42.8007 52.4013 42.6098 52.6486 42.3594 52.8233C42.1089 52.998 41.8109 53.0917 41.5055 53.0917C41.2002 53.0917 40.9022 52.998 40.6517 52.8233C40.4012 52.6486 40.2104 52.4013 40.1049 52.1148L38.3124 47.2754L33.473 45.4941C33.1865 45.3887 32.9391 45.1978 32.7645 44.9474C32.5898 44.6969 32.4961 44.3989 32.4961 44.0935C32.4961 43.7882 32.5898 43.4901 32.7645 43.2397C32.9391 42.9892 33.1865 42.7984 33.473 42.6929L38.3124 40.9004L40.0937 36.061C40.1991 35.7744 40.39 35.5271 40.6404 35.3524C40.8909 35.1777 41.1889 35.0841 41.4943 35.0841C41.7996 35.0841 42.0977 35.1777 42.3481 35.3524C42.5986 35.5271 42.7894 35.7744 42.8949 36.061L44.6874 40.9004L49.5268 42.6816C49.8146 42.7864 50.0628 42.978 50.2371 43.2299C50.4114 43.4818 50.5033 43.7816 50.4999 44.0879ZM45.2499 35.0879H46.7499V36.5879C46.7499 36.7868 46.8289 36.9776 46.9696 37.1182C47.1102 37.2589 47.301 37.3379 47.4999 37.3379C47.6988 37.3379 47.8896 37.2589 48.0302 37.1182C48.1709 36.9776 48.2499 36.7868 48.2499 36.5879V35.0879H49.7499C49.9488 35.0879 50.1396 35.0089 50.2802 34.8682C50.4209 34.7276 50.4999 34.5368 50.4999 34.3379C50.4999 34.139 50.4209 33.9482 50.2802 33.8076C50.1396 33.6669 49.9488 33.5879 49.7499 33.5879H48.2499V32.0879C48.2499 31.889 48.1709 31.6982 48.0302 31.5576C47.8896 31.4169 47.6988 31.3379 47.4999 31.3379C47.301 31.3379 47.1102 31.4169 46.9696 31.5576C46.8289 31.6982 46.7499 31.889 46.7499 32.0879V33.5879H45.2499C45.051 33.5879 44.8602 33.6669 44.7196 33.8076C44.5789 33.9482 44.4999 34.139 44.4999 34.3379C44.4999 34.5368 44.5789 34.7276 44.7196 34.8682C44.8602 35.0089 45.051 35.0879 45.2499 35.0879ZM53.4999 38.0879H52.7499V37.3379C52.7499 37.139 52.6709 36.9482 52.5302 36.8076C52.3896 36.6669 52.1988 36.5879 51.9999 36.5879C51.801 36.5879 51.6102 36.6669 51.4696 36.8076C51.3289 36.9482 51.2499 37.139 51.2499 37.3379V38.0879H50.4999C50.301 38.0879 50.1102 38.1669 49.9696 38.3076C49.8289 38.4482 49.7499 38.639 49.7499 38.8379C49.7499 39.0368 49.8289 39.2276 49.9696 39.3682C50.1102 39.5089 50.301 39.5879 50.4999 39.5879H51.2499V40.3379C51.2499 40.5368 51.3289 40.7276 51.4696 40.8682C51.6102 41.0089 51.801 41.0879 51.9999 41.0879C52.1988 41.0879 52.3896 41.0089 52.5302 40.8682C52.6709 40.7276 52.7499 40.5368 52.7499 40.3379V39.5879H53.4999C53.6988 39.5879 53.8896 39.5089 54.0302 39.3682C54.1709 39.2276 54.2499 39.0368 54.2499 38.8379C54.2499 38.639 54.1709 38.4482 54.0302 38.3076C53.8896 38.1669 53.6988 38.0879 53.4999 38.0879Z" fill="white"/>
+</svg>
diff --git a/docs-website/static/img/solutions/unified-tab-detect.png b/docs-website/static/img/solutions/unified-tab-detect.png
new file mode 100644
index 0000000000000..ae178862aa48b
Binary files /dev/null and b/docs-website/static/img/solutions/unified-tab-detect.png differ
diff --git a/docs-website/static/img/solutions/unified-tab-resolve.png b/docs-website/static/img/solutions/unified-tab-resolve.png
new file mode 100644
index 0000000000000..6f116192fd792
Binary files /dev/null and b/docs-website/static/img/solutions/unified-tab-resolve.png differ
diff --git a/docs-website/static/img/solutions/unified-tab-unify.png b/docs-website/static/img/solutions/unified-tab-unify.png
new file mode 100644
index 0000000000000..0f01b88b0cfde
Binary files /dev/null and b/docs-website/static/img/solutions/unified-tab-unify.png differ
diff --git a/docs-website/static/img/solutions/water-lock.png b/docs-website/static/img/solutions/water-lock.png
new file mode 100644
index 0000000000000..a7fac5bdcb719
Binary files /dev/null and b/docs-website/static/img/solutions/water-lock.png differ
diff --git a/docs-website/static/img/solutions/wolt.png b/docs-website/static/img/solutions/wolt.png
new file mode 100644
index 0000000000000..2633136f36ed7
Binary files /dev/null and b/docs-website/static/img/solutions/wolt.png differ
diff --git a/docs-website/vercel-setup.sh b/docs-website/vercel-setup.sh
index 4bb40eaddf477..e9ba87b75be77 100755
--- a/docs-website/vercel-setup.sh
+++ b/docs-website/vercel-setup.sh
@@ -5,8 +5,8 @@ set -euxo pipefail
 ./metadata-ingestion/scripts/install_deps.sh
 
 # Set up java version for gradle
-yum install java-17-amazon-corretto -y
-java --version
+yum install java-17-amazon-corretto-devel -y
+javac --version
 
 # Build python from source.
 # Amazon Linux 2 has Python 3.8, but it's version of OpenSSL is super old and hence it
diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock
index 039dac5d5556b..bad57c8c6c900 100644
--- a/docs-website/yarn.lock
+++ b/docs-website/yarn.lock
@@ -1827,7 +1827,7 @@
     "@docusaurus/theme-search-algolia" "2.4.3"
     "@docusaurus/types" "2.4.3"
 
-"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2":
+"@docusaurus/react-loadable@5.5.2":
   version "5.5.2"
   resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce"
   integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==
@@ -2843,6 +2843,11 @@
     rc-resize-observer "^1.3.1"
     rc-util "^5.38.0"
 
+"@servicebell/widget@^0.1.6":
+  version "0.1.6"
+  resolved "https://registry.yarnpkg.com/@servicebell/widget/-/widget-0.1.6.tgz#04672a7e7b14ff7025ec83fd740373345c359d74"
+  integrity sha512-Kh21FAETJlk32MRXHGYKxFTWLbbLlgLX3lKYH/02KHaQGIPRdHh/Ok7DiRc+/6f/OTQq3x3ady7e4o9weV5yQg==
+
 "@sideway/address@^4.1.5":
   version "4.1.5"
   resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.5.tgz#4bc149a0076623ced99ca8208ba780d65a99b9d5"
@@ -4752,9 +4757,9 @@ cross-fetch@^3.1.5:
     node-fetch "^2.6.12"
 
 cross-spawn@^7.0.3:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
+  integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
   dependencies:
     path-key "^3.1.0"
     shebang-command "^2.0.0"
@@ -6347,9 +6352,9 @@ http-proxy-agent@^7.0.0, http-proxy-agent@^7.0.1:
     debug "^4.3.4"
 
 http-proxy-middleware@^2.0.3:
-  version "2.0.6"
-  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz#e1a4dd6979572c7ab5a4e4b55095d1f32a74963f"
-  integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6"
+  integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==
   dependencies:
     "@types/http-proxy" "^1.17.8"
     http-proxy "^1.18.1"
@@ -8257,9 +8262,9 @@ multicast-dns@^7.2.5:
     thunky "^1.0.2"
 
 nanoid@^3.3.7:
-  version "3.3.7"
-  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8"
-  integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==
+  version "3.3.8"
+  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf"
+  integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==
 
 napi-build-utils@^1.0.1:
   version "1.0.2"
@@ -9708,6 +9713,14 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1:
   dependencies:
     "@babel/runtime" "^7.10.3"
 
+"react-loadable@npm:@docusaurus/react-loadable@5.5.2":
+  version "5.5.2"
+  resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce"
+  integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==
+  dependencies:
+    "@types/react" "*"
+    prop-types "^15.6.2"
+
 react-markdown@^8.0.6:
   version "8.0.7"
   resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b"
@@ -9822,6 +9835,11 @@ react-textarea-autosize@^8.3.2:
     use-composed-ref "^1.3.0"
     use-latest "^1.2.1"
 
+react-use-draggable-scroll@^0.4.7:
+  version "0.4.7"
+  resolved "https://registry.yarnpkg.com/react-use-draggable-scroll/-/react-use-draggable-scroll-0.4.7.tgz#86e77caab921ca07b134e9e1d1bc1810aeee4916"
+  integrity sha512-6gCxGPO9WV5dIsBaDrgUKBaac8CY07PkygcArfajijYSNDwAq0girDRjaBuF1+lRqQryoLFQfpVaV2u/Yh6CrQ==
+
 react-waypoint@^10.3.0:
   version "10.3.0"
   resolved "https://registry.yarnpkg.com/react-waypoint/-/react-waypoint-10.3.0.tgz#fcc60e86c6c9ad2174fa58d066dc6ae54e3df71d"
diff --git a/docs/advanced/bootstrap-mcps.md b/docs/advanced/bootstrap-mcps.md
index 0aa4b7608740f..c3ad7db2016db 100644
--- a/docs/advanced/bootstrap-mcps.md
+++ b/docs/advanced/bootstrap-mcps.md
@@ -149,6 +149,28 @@ to the required json structure and stored as a string.
       executorId: default
 ```
 
+## `bootstrap_mcps.yaml` Override
+
+Additionally, the `bootstrap_mcps.yaml` can be overridden.
+This might be useful for applying changes to the version when using helm defined template values.
+
+```yaml
+bootstrap:
+  templates:
+    - name: myMCPTemplate
+      version: v1
+      mcps_location: <classpath or file location>
+      values_env: <value environment variable>
+      revision_env: REVISION_ENV
+```
+
+In the above example, we've added a `revision_env` which allows overriding the MCP bootstrap definition itself (excluding `revision_env`).
+
+In this example we could configure `REVISION_ENV` to contain a timestamp or hash: `{"version":"2024060600"}` 
+This value can be changed/incremented each time the helm supplied template values change. This ensures the MCP is updated
+with the latest values during deployment.
+
+
 ## Known Limitations
 
 * Supported change types: 
diff --git a/docs/advanced/mcp-mcl.md b/docs/advanced/mcp-mcl.md
index 9efb9b794954d..3a06b2abadc11 100644
--- a/docs/advanced/mcp-mcl.md
+++ b/docs/advanced/mcp-mcl.md
@@ -14,6 +14,18 @@ To mitigate these downsides, we are committed to providing cross-language client
 
 Ultimately, we intend to realize a state in which the Entities and Aspect schemas can be altered without requiring generated code and without maintaining a single mega-model schema (looking at you, Snapshot.pdl). The intention is that changes to the metadata model become even easier than they are today.
 
+### Synchronous Ingestion Architecture
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/advanced/mcp-mcl/sync-ingestion.svg"/>
+</p>
+
+### Asynchronous Ingestion Architecture
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/advanced/mcp-mcl/async-ingestion.svg"/>
+</p>
+
 ## Modeling
 
 A Metadata Change Proposal is defined (in PDL) as follows
@@ -206,3 +218,6 @@ Another form of conditional writes which considers the existence of an aspect or
 
 `CREATE_ENTITY` - Create the aspect if no aspects exist for the entity.
 
+By default, a validation exception is thrown if the `CREATE`/`CREATE_ENTITY` constraint is violated. If the write operation
+should be dropped without considering it an exception, then add the following header: `If-None-Match: *` to the MCP.
+
diff --git a/docs/advanced/patch.md b/docs/advanced/patch.md
index 601d055659313..24e8c68a9168d 100644
--- a/docs/advanced/patch.md
+++ b/docs/advanced/patch.md
@@ -1,69 +1,120 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# But First, Semantics: Upsert versus Patch
+# Emitting Patch Updates to DataHub
 
 ## Why Would You Use Patch
 
-By default, most of the SDK tutorials and API-s involve applying full upserts at the aspect level. This means that typically, when you want to change one field within an aspect without modifying others, you need to do a read-modify-write to not overwrite existing fields.
-To support these scenarios, DataHub supports PATCH based operations so that targeted changes to single fields or values within arrays of fields are possible without impacting other existing metadata.
+By default, most of the SDK tutorials and APIs involve applying full upserts at the aspect level, e.g. replacing the aspect entirely. 
+This means that when you want to change even a single field within an aspect without modifying others, you need to do a read-modify-write to avoid overwriting existing fields.
+To support these scenarios, DataHub supports `PATCH` operations to perform targeted changes for individual fields or values within arrays of fields are possible without impacting other existing metadata.
 
 :::note
 
-Currently, PATCH support is only available for a selected set of aspects, so before pinning your hopes on using PATCH as a way to make modifications to aspect values, confirm whether your aspect supports PATCH semantics. The complete list of Aspects that are supported are maintained [here](https://github.com/datahub-project/datahub/blob/9588440549f3d99965085e97b214a7dabc181ed2/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java#L24). In the near future, we do have plans to automatically support PATCH semantics for aspects by default.
+Currently, PATCH support is only available for a selected set of aspects, so before pinning your hopes on using PATCH as a way to make modifications to aspect values, confirm whether your aspect supports PATCH semantics. The complete list of Aspects that are supported are maintained [here](https://github.com/datahub-project/datahub/blob/9588440549f3d99965085e97b214a7dabc181ed2/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java#L24). 
 
 :::
 
-## How To Use Patch
+## How To Use Patches
 
-Examples for using Patch are sprinkled throughout the API guides.
 Here's how to find the appropriate classes for the language for your choice.
 
-
 <Tabs>
-<TabItem value="Java" label="Java SDK">
+<TabItem value="Python" label="Python SDK" default>
 
-The Java Patch builders are aspect-oriented and located in the [datahub-client](https://github.com/datahub-project/datahub/tree/master/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch) module under the `datahub.client.patch` namespace.
+The Python Patch builders are entity-oriented and located in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/9588440549f3d99965085e97b214a7dabc181ed2/metadata-ingestion/src/datahub/specific) module and located in the `datahub.specific` module. 
+Patch builder helper classes exist for 
 
-Here are a few illustrative examples using the Java Patch builders:
+- [Datasets](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/dataset.py)
+- [Charts](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/chart.py)
+- [Dashboards](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/dashboard.py)
+- [Data Jobs (Tasks)](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/datajob.py)
+- [Data Products](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/dataproduct.py)
 
+And we are gladly accepting contributions for Containers, Data Flows (Pipelines), Tags, Glossary Terms, Domains, and ML Models.
 
-### Add Custom Properties
+### Add & Remove Owners for Dataset
 
-```java
-{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAdd.java show_path_as_comment }}
+To add & remove specific owners for a dataset:
+
+```python
+{{ inline /metadata-ingestion/examples/library/dataset_add_owner_patch.py show_path_as_comment }}
 ```
 
-### Add and Remove Custom Properties
+### Add & Remove Tags for Dataset
 
-```java
-{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java show_path_as_comment }}
+To add & remove specific tags for a dataset:
+
+```python
+{{ inline /metadata-ingestion/examples/library/dataset_add_tag_patch.py show_path_as_comment }}
 ```
 
-### Add Data Job Lineage
+And for a specific schema field within the Dataset:
 
-```java
-{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java show_path_as_comment }}
+```python
+{{ inline /metadata-ingestion/examples/library/dataset_field_add_tag_patch.py show_path_as_comment }}
 ```
 
-</TabItem>
-<TabItem value="Python" label="Python SDK" default>
+### Add & Remove Glossary Terms for Dataset
+
+To add & remove specific glossary terms for a dataset:
+
+```python
+{{ inline /metadata-ingestion/examples/library/dataset_add_glossary_term_patch.py show_path_as_comment }}
+```
+
+And for a specific schema field within the Dataset:
+
+```python
+{{ inline /metadata-ingestion/examples/library/dataset_field_add_glossary_term_patch.py show_path_as_comment }}
+```
+
+### Add & Remove Structured Properties for Dataset
 
-The Python Patch builders are entity-oriented and located in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/9588440549f3d99965085e97b214a7dabc181ed2/metadata-ingestion/src/datahub/specific) module and located in the `datahub.specific` module.
+To add & remove structured properties for a dataset:
 
-Here are a few illustrative examples using the Python Patch builders:
+```python
+{{ inline /metadata-ingestion/examples/library/dataset_add_structured_properties_patch.py show_path_as_comment }}
+```
 
-### Add Properties to Dataset
+### Add & Remove Upstream Lineage for Dataset
+
+To add & remove a lineage edge connecting a dataset to it's upstream or input at both the dataset and schema field level:
 
 ```python
-{{ inline /metadata-ingestion/examples/library/dataset_add_properties.py show_path_as_comment }}
+{{ inline /metadata-ingestion/examples/library/dataset_add_upstream_lineage_patch.py show_path_as_comment }}
+```
+
+### Add & Remove Read-Only Custom Properties for Dataset
+
+To add & remove specific custom properties for a dataset: 
+
+```python
+{{ inline /metadata-ingestion/examples/library/dataset_add_remove_custom_properties_patch.py show_path_as_comment }}
+```
+
+</TabItem>
+<TabItem value="Java" label="Java SDK">
+
+The Java Patch builders are aspect-oriented and located in the [datahub-client](https://github.com/datahub-project/datahub/tree/master/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch) module under the `datahub.client.patch` namespace.
+
+### Add & Remove Read-Only Custom Properties
+
+```java
+{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java show_path_as_comment }}
+```
+
+### Add Data Job Lineage
+
+```java
+{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java show_path_as_comment }}
 ```
 
 </TabItem>
 </Tabs>
 
 
-## How Patch works
+## Advanced: How Patch works
 
 To understand how patching works, it's important to understand a bit about our [models](../what/aspect.md). Entities are comprised of Aspects
 which can be reasoned about as JSON representations of the object models. To be able to patch these we utilize [JsonPatch](https://jsonpatch.com/). The components of a JSON Patch are the path, operation, and value.
@@ -73,9 +124,6 @@ which can be reasoned about as JSON representations of the object models. To be
 The JSON path refers to a value within the schema. This can be a single field or can be an entire object reference depending on what the path is.
 For our patches we are primarily targeting single fields or even single array elements within a field. To be able to target array elements by id, we go through a translation process
 of the schema to transform arrays into maps. This allows a path to reference a particular array element by key rather than by index, for example a specific tag urn being added to a dataset.
-This is important to note that for some fields in our schema that are arrays which do not necessarily restrict uniqueness, this puts a uniqueness constraint on the key.
-The key for objects stored in arrays is determined manually by examining the schema and a long term goal is to make these keys annotation driven to reduce the amount of code needed to support
-additional aspects to be patched. There is a generic patch endpoint, but it requires any array field keys to be specified at request time, putting a lot of burden on the API user.
 
 #### Examples
 
@@ -87,8 +135,7 @@ Breakdown:
 * `/upstreams` -> References the upstreams field of the UpstreamLineage aspect, this is an array of Upstream objects where the key is the Urn
 * `/urn:...` -> The dataset to be targeted by the operation
 
-
-A patch path for targeting a fine grained lineage upstream:
+A patch path for targeting a fine-grained lineage upstream:
 
 `/fineGrainedLineages/TRANSFORM/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),foo)/urn:li:query:queryId/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD),bar)`
 
@@ -118,7 +165,6 @@ using adds, but generally the most useful use case for patch is to add elements
 
 Remove operations require the path specified to be present, or an error will be thrown, otherwise they operate as one would expect. The specified path will be removed from the aspect.
 
-
 ### Value
 
 Value is the actual information that will be stored at a path. If the path references an object then this will include the JSON key value pairs for that object.
diff --git a/docs/api/datahub-apis.md b/docs/api/datahub-apis.md
index 6bb793a59a86e..c46aacde3a0cb 100644
--- a/docs/api/datahub-apis.md
+++ b/docs/api/datahub-apis.md
@@ -2,18 +2,16 @@
 
 DataHub has several APIs to manipulate metadata on the platform. Here's the list of APIs and their pros and cons to help you choose the right one for your use case.
 
-| API                                                                            | Definition                         | Pros                                     | Cons                                                        |
-|--------------------------------------------------------------------------------|------------------------------------|------------------------------------------|-------------------------------------------------------------| 
-| **[Python SDK](/metadata-ingestion/as-a-library.md)**                          | SDK                                | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event      |  
-| **[Java SDK](/metadata-integration/java/as-a-library.md)**                     | SDK                                | Highly flexible, Good for bulk execution                        | Requires an understanding of the metadata change event      | 
-| **[GraphQL API](docs/api/graphql/getting-started.md)**                         | GraphQL interface                  | Intuitive; mirrors UI capabilities       | Less flexible than SDKs; requires knowledge of GraphQL syntax | 
-| **[OpenAPI](docs/api/openapi/openapi-usage-guide.md)** <br /> (Not&nbsp;Recommended) | Lower-level API for advanced users |                                          | Generally not recommended for typical use cases             | 
+| API                                                        | Definition                         | Pros                                     | Cons                                                                    |
+| ---------------------------------------------------------- | ---------------------------------- | ---------------------------------------- | ----------------------------------------------------------------------- |
+| **[Python SDK](/metadata-ingestion/as-a-library.md)**      | SDK                                | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event                  |
+| **[Java SDK](/metadata-integration/java/as-a-library.md)** | SDK                                | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event                  |
+| **[GraphQL API](docs/api/graphql/getting-started.md)**     | GraphQL interface                  | Intuitive; mirrors UI capabilities       | Less flexible than SDKs; requires knowledge of GraphQL syntax           |
+| **[OpenAPI](docs/api/openapi/openapi-usage-guide.md)**     | Lower-level API for advanced users | Most powerful and flexible               | Can be hard to use for straightforward use cases; no corresponding SDKs |
 
 In general, **Python and Java SDKs** are our most recommended tools for extending and customizing the behavior of your DataHub instance.
 We don't recommend using the **OpenAPI** directly, as it's more complex and less user-friendly than the other APIs.
 
-
-
 ## Python and Java SDK
 
 We offer an SDK for both Python and Java that provide full functionality when it comes to CRUD operations and any complex functionality you may want to build into DataHub. We recommend using the SDKs for most use cases. Here are the examples of how to use the SDKs:
@@ -23,22 +21,22 @@ We offer an SDK for both Python and Java that provide full functionality when it
 - Creating custom metadata entities
 
 Learn more about the SDKs:
+
 - **[Python SDK →](/metadata-ingestion/as-a-library.md)**
 - **[Java SDK →](/metadata-integration/java/as-a-library.md)**
 
-
 ## GraphQL API
 
 The `graphql` API serves as the primary public API for the platform. It can be used to fetch and update metadata programatically in the language of your choice. Intended as a higher-level API that simplifies the most common operations.
 
 We recommend using the GraphQL API if you're getting started with DataHub since it's more user-friendly and straighfowrad. Here are some examples of how to use the GraphQL API:
+
 - Search for datasets with conditions
 - Update a certain field of a dataset
 
 Learn more about the GraphQL API:
-- **[GraphQL API →](docs/api/graphql/getting-started.md)**
-
 
+- **[GraphQL API →](docs/api/graphql/getting-started.md)**
 
 ## DataHub API Comparison
 
@@ -47,59 +45,59 @@ Here's an overview of what each API can do.
 
 > Last Updated : Feb 16 2024
 
-| Feature                            | GraphQL                                                                      | Python SDK                                                                   | OpenAPI |
-|------------------------------------|------------------------------------------------------------------------------|------------------------------------------------------------------------------|---------|
-| Create a Dataset                   | 🚫                                                                           | ✅ [[Guide]](/docs/api/tutorials/datasets.md)                                 | ✅       |
-| Delete a Dataset (Soft Delete)     | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset)                  | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset)                  | ✅       |
-| Delete a Dataset (Hard Delete)     | 🚫                                                                           | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset)                  | ✅       |
-| Search a Dataset                   | ✅ [[Guide]](/docs/how/search.md#graphql)                                     | ✅                                                                            | ✅       |
-| Read a Dataset Deprecation         | ✅                                                                            | ✅                                                                            | ✅       |
-| Read Dataset Entities (V2)         | ✅                                                                            | ✅                                                                            | ✅       |
-| Create a Tag                       | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags)                         | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags)                         | ✅       |
-| Read a Tag                         | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags)                           | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags)                           | ✅       |
-| Add Tags to a Dataset              | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset)               | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset)               | ✅       |
-| Add Tags to a Column of a Dataset  | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset)   | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset)   | ✅       |
-| Remove Tags from a Dataset         | ✅ [[Guide]](/docs/api/tutorials/tags.md#remove-tags)                         | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags#remove-tags)                | ✅       |
-| Create Glossary Terms              | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms)                       | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms)                       | ✅       |
-| Read Terms from a Dataset          | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms)                         | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms)                         | ✅       |
-| Add Terms to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅       |
-| Add Terms to a Dataset             | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset)             | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset)             | ✅       |
-| Create Domains                     | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain)                    | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain)                    | ✅       |
-| Read Domains                       | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains)                     | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains)                     | ✅       |
-| Add Domains to a Dataset           | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains)                      | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains)                      | ✅       |
-| Remove Domains from a Dataset      | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains)                   | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains)                   | ✅       |
-| Create / Upsert Users              | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users)                      | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users)                      | ✅       |
-| Create / Upsert Group              | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group)                      | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group)                      | ✅       |
-| Read Owners of a Dataset           | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners)                       | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners)                       | ✅       |
-| Add Owner to a Dataset             | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners)                        | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners#remove-owners)          | ✅       |
-| Remove Owner from a Dataset        | ✅ [[Guide]](/docs/api/tutorials/owners.md#remove-owners)                     | ✅ [[Guide]](/docs/api/tutorials/owners.md)                                   | ✅       |
-| Add Lineage                        | ✅ [[Guide]](/docs/api/tutorials/lineage.md)                     | ✅ [[Guide]](/docs/api/tutorials/lineage.md#add-lineage)                      | ✅ |
-| Add Column Level (Fine Grained) Lineage                  | 🚫                                                            | ✅ [[Guide]](docs/api/tutorials/lineage.md#add-column-level-lineage)                                                           | ✅       |
-| Add Documentation (Description) to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅       |
-| Add Documentation (Description) to a Dataset             | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅       |
-| Add / Remove / Replace Custom Properties on a Dataset    | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/custom-properties.md)        | ✅       |
-| Add ML Feature to ML Feature Table                       | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlfeaturetable)        | ✅       |
-| Add ML Feature to MLModel                                | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlmodel)        | ✅       |
-| Add ML Group to MLFeatureTable                           | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlgroup-to-mlfeaturetable)        | ✅       |
-| Create MLFeature                                         | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeature)      | ✅       |
-| Create MLFeatureTable                                    | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable) | ✅       |
-| Create MLModel                                           | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodel)        | ✅       |
-| Create MLModelGroup                                      | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodelgroup)   | ✅       |
-| Create MLPrimaryKey                                      | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlprimarykey)   | ✅       |
-| Create MLFeatureTable                                    | 🚫                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable)| ✅       |
-| Read MLFeature                                           | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature)        | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature)        | ✅       |
-| Read MLFeatureTable                                      | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable)   | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable)   | ✅       |
-| Read MLModel                                             | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel)          | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel)          | ✅       |
-| Read MLModelGroup                                        | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup)     | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup)     | ✅       |
-| Read MLPrimaryKey                                        | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey)     | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey)     | ✅       |
-| Create Data Product                                      | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/create_dataproduct.py)        | ✅       |
-| Create Lineage Between Chart and Dashboard               | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_chart_dashboard.py) | ✅       |
-| Create Lineage Between Dataset and Chart                 | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_chart.py) | ✅       |
-| Create Lineage Between Dataset and DataJob               | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_job_dataset.py) | ✅       |
-| Create Finegrained Lineage as DataJob for Dataset        | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_datajob_finegrained.py) | ✅       |
-| Create Finegrained Lineage for Dataset                  | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_dataset_finegrained.py)        | ✅       |
-| Create Dataset Lineage with Kafka                       | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_kafka.py)        | ✅       |
-| Create Dataset Lineage with MCPW & Rest Emitter         | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py)        | ✅       |
-| Create Dataset Lineage with Rest Emitter                | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_rest.py)        | ✅       |
-| Create DataJob with Dataflow                            | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow.py) [[Simple]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py) [[Verbose]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_verbose.py) | ✅       |
-| Create Programmatic Pipeline                            | 🚫                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/programatic_pipeline.py) | ✅       |
+| Feature                                                  | GraphQL                                                                       | Python SDK                                                                                                                                                                                                                                                                                                                                                                                                                    | OpenAPI |
+| -------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
+| Create a Dataset                                         | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/datasets.md)                                                                                                                                                                                                                                                                                                                                                                                 | ✅      |
+| Delete a Dataset (Soft Delete)                           | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset)                  | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset)                                                                                                                                                                                                                                                                                                                                                                  | ✅      |
+| Delete a Dataset (Hard Delete)                           | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset)                                                                                                                                                                                                                                                                                                                                                                  | ✅      |
+| Search a Dataset                                         | ✅ [[Guide]](/docs/how/search.md#graphql)                                     | ✅                                                                                                                                                                                                                                                                                                                                                                                                                            | ✅      |
+| Read a Dataset Deprecation                               | ✅                                                                            | ✅                                                                                                                                                                                                                                                                                                                                                                                                                            | ✅      |
+| Read Dataset Entities (V2)                               | ✅                                                                            | ✅                                                                                                                                                                                                                                                                                                                                                                                                                            | ✅      |
+| Create a Tag                                             | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags)                         | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags)                                                                                                                                                                                                                                                                                                                                                                         | ✅      |
+| Read a Tag                                               | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags)                           | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags)                                                                                                                                                                                                                                                                                                                                                                           | ✅      |
+| Add Tags to a Dataset                                    | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset)               | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset)                                                                                                                                                                                                                                                                                                                                                               | ✅      |
+| Add Tags to a Column of a Dataset                        | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset)   | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset)                                                                                                                                                                                                                                                                                                                                                   | ✅      |
+| Remove Tags from a Dataset                               | ✅ [[Guide]](/docs/api/tutorials/tags.md#remove-tags)                         | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags#remove-tags)                                                                                                                                                                                                                                                                                                                                                                | ✅      |
+| Create Glossary Terms                                    | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms)                       | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms)                                                                                                                                                                                                                                                                                                                                                                       | ✅      |
+| Read Terms from a Dataset                                | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms)                         | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms)                                                                                                                                                                                                                                                                                                                                                                         | ✅      |
+| Add Terms to a Column of a Dataset                       | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset)                                                                                                                                                                                                                                                                                                                                                 | ✅      |
+| Add Terms to a Dataset                                   | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset)             | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset)                                                                                                                                                                                                                                                                                                                                                             | ✅      |
+| Create Domains                                           | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain)                    | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain)                                                                                                                                                                                                                                                                                                                                                                    | ✅      |
+| Read Domains                                             | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains)                     | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains)                                                                                                                                                                                                                                                                                                                                                                     | ✅      |
+| Add Domains to a Dataset                                 | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains)                      | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains)                                                                                                                                                                                                                                                                                                                                                                      | ✅      |
+| Remove Domains from a Dataset                            | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains)                   | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains)                                                                                                                                                                                                                                                                                                                                                                   | ✅      |
+| Create / Upsert Users                                    | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users)                      | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users)                                                                                                                                                                                                                                                                                                                                                                      | ✅      |
+| Create / Upsert Group                                    | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group)                      | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group)                                                                                                                                                                                                                                                                                                                                                                      | ✅      |
+| Read Owners of a Dataset                                 | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners)                       | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners)                                                                                                                                                                                                                                                                                                                                                                       | ✅      |
+| Add Owner to a Dataset                                   | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners)                        | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners#remove-owners)                                                                                                                                                                                                                                                                                                                                                          | ✅      |
+| Remove Owner from a Dataset                              | ✅ [[Guide]](/docs/api/tutorials/owners.md#remove-owners)                     | ✅ [[Guide]](/docs/api/tutorials/owners.md)                                                                                                                                                                                                                                                                                                                                                                                   | ✅      |
+| Add Lineage                                              | ✅ [[Guide]](/docs/api/tutorials/lineage.md)                                  | ✅ [[Guide]](/docs/api/tutorials/lineage.md#add-lineage)                                                                                                                                                                                                                                                                                                                                                                      | ✅      |
+| Add Column Level (Fine Grained) Lineage                  | 🚫                                                                            | ✅ [[Guide]](docs/api/tutorials/lineage.md#add-column-level-lineage)                                                                                                                                                                                                                                                                                                                                                          | ✅      |
+| Add Documentation (Description) to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column)   | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column)                                                                                                                                                                                                                                                                                                                                                   | ✅      |
+| Add Documentation (Description) to a Dataset             | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset)  | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset)                                                                                                                                                                                                                                                                                                                                                  | ✅      |
+| Add / Remove / Replace Custom Properties on a Dataset    | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/custom-properties.md)                                                                                                                                                                                                                                                                                                                                                                        | ✅      |
+| Add ML Feature to ML Feature Table                       | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlfeaturetable)                                                                                                                                                                                                                                                                                                                                                       | ✅      |
+| Add ML Feature to MLModel                                | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlmodel)                                                                                                                                                                                                                                                                                                                                                              | ✅      |
+| Add ML Group to MLFeatureTable                           | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlgroup-to-mlfeaturetable)                                                                                                                                                                                                                                                                                                                                                         | ✅      |
+| Create MLFeature                                         | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeature)                                                                                                                                                                                                                                                                                                                                                                      | ✅      |
+| Create MLFeatureTable                                    | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable)                                                                                                                                                                                                                                                                                                                                                                 | ✅      |
+| Create MLModel                                           | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodel)                                                                                                                                                                                                                                                                                                                                                                        | ✅      |
+| Create MLModelGroup                                      | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodelgroup)                                                                                                                                                                                                                                                                                                                                                                   | ✅      |
+| Create MLPrimaryKey                                      | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlprimarykey)                                                                                                                                                                                                                                                                                                                                                                   | ✅      |
+| Create MLFeatureTable                                    | 🚫                                                                            | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable)                                                                                                                                                                                                                                                                                                                                                                 | ✅      |
+| Read MLFeature                                           | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature)                        | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature)                                                                                                                                                                                                                                                                                                                                                                        | ✅      |
+| Read MLFeatureTable                                      | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable)                   | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable)                                                                                                                                                                                                                                                                                                                                                                   | ✅      |
+| Read MLModel                                             | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel)                          | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel)                                                                                                                                                                                                                                                                                                                                                                          | ✅      |
+| Read MLModelGroup                                        | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup)                     | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup)                                                                                                                                                                                                                                                                                                                                                                     | ✅      |
+| Read MLPrimaryKey                                        | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey)                     | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey)                                                                                                                                                                                                                                                                                                                                                                     | ✅      |
+| Create Data Product                                      | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/create_dataproduct.py)                                                                                                                                                                                                                                                                                                 | ✅      |
+| Create Lineage Between Chart and Dashboard               | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_chart_dashboard.py)                                                                                                                                                                                                                                                                                            | ✅      |
+| Create Lineage Between Dataset and Chart                 | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_chart.py)                                                                                                                                                                                                                                                                                              | ✅      |
+| Create Lineage Between Dataset and DataJob               | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_job_dataset.py)                                                                                                                                                                                                                                                                                        | ✅      |
+| Create Finegrained Lineage as DataJob for Dataset        | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_datajob_finegrained.py)                                                                                                                                                                                                                                                                                | ✅      |
+| Create Finegrained Lineage for Dataset                   | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_dataset_finegrained.py)                                                                                                                                                                                                                                                                                | ✅      |
+| Create Dataset Lineage with Kafka                        | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_kafka.py)                                                                                                                                                                                                                                                                                              | ✅      |
+| Create Dataset Lineage with MCPW & Rest Emitter          | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py)                                                                                                                                                                                                                                                                                          | ✅      |
+| Create Dataset Lineage with Rest Emitter                 | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_rest.py)                                                                                                                                                                                                                                                                                               | ✅      |
+| Create DataJob with Dataflow                             | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow.py) [[Simple]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py) [[Verbose]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_verbose.py) | ✅      |
+| Create Programmatic Pipeline                             | 🚫                                                                            | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/programatic_pipeline.py)                                                                                                                                                                                                                                                                                               | ✅      |
diff --git a/docs/api/restli/restli-overview.md b/docs/api/restli/restli-overview.md
index 22b913d9a25df..3e9ab00b52267 100644
--- a/docs/api/restli/restli-overview.md
+++ b/docs/api/restli/restli-overview.md
@@ -1156,7 +1156,7 @@ curl -X POST 'http://localhost:8080/entities?action=search' \
             "and": [
                {
                   "field": "title",
-                  "value": "Baz Chart 1",
+                  "values": ["Baz Chart 1"],
                   "condition": "EQUAL"
                }
             ]   
@@ -1261,7 +1261,7 @@ curl -X POST 'http://localhost:8080/entities?action=autocomplete' \
             "and": [
                {
                 "field": "tool",
-                "value": "looker",
+                "values": ["looker"],
                 "condition": "EQUAL"
                }
             ]   
diff --git a/docs/api/tutorials/custom-assertions.md b/docs/api/tutorials/custom-assertions.md
index 6544efb8809c2..47975c5739464 100644
--- a/docs/api/tutorials/custom-assertions.md
+++ b/docs/api/tutorials/custom-assertions.md
@@ -265,7 +265,7 @@ query getAssertion {
             customType # Will be your custom type. 
             description
             lastUpdated {
-                time
+                time 
                 actor
             }
             customAssertion {
@@ -282,6 +282,18 @@ query getAssertion {
                 }
             }
         }
+        # Fetch what entities have the assertion attached to it
+        relationships(input: {
+            types: ["Asserts"]
+            direction: OUTGOING
+        }) {
+            total
+            relationships {
+                entity {
+                    urn
+                }
+            }
+        }
     }
 }
 ```
diff --git a/docs/api/tutorials/custom-properties.md b/docs/api/tutorials/custom-properties.md
index fe0d7e62dcde8..86b1b2c0c54da 100644
--- a/docs/api/tutorials/custom-properties.md
+++ b/docs/api/tutorials/custom-properties.md
@@ -74,7 +74,7 @@ The following code adds custom properties `cluster_name` and `retention_time` to
 <TabItem value="python" label="Python" default>
 
 ```python
-{{ inline /metadata-ingestion/examples/library/dataset_add_properties.py show_path_as_comment }}
+{{ inline /metadata-ingestion/examples/library/dataset_add_custom_properties_patch.py show_path_as_comment }}
 ```
 
 </TabItem>
@@ -128,7 +128,7 @@ The following code shows you how can add and remove custom properties in the sam
 <TabItem value="python" label="Python" default>
 
 ```python
-{{ inline /metadata-ingestion/examples/library/dataset_add_remove_properties.py show_path_as_comment }}
+{{ inline /metadata-ingestion/examples/library/dataset_add_remove_custom_properties_patch.py show_path_as_comment }}
 ```
 
 </TabItem>
diff --git a/docs/api/tutorials/forms.md b/docs/api/tutorials/forms.md
index cf51f1579f1c8..30dd4db7d8f11 100644
--- a/docs/api/tutorials/forms.md
+++ b/docs/api/tutorials/forms.md
@@ -1,13 +1,15 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Documentation Forms
+# Compliance Forms
 
-## Why Would You Use Documentation Forms?
+## Why Would You Use Compliance Forms?
 
-Documentation Forms are a way for end-users to fill out all mandatory attributes associated with a data asset. The form will be dynamically generated based on the definitions provided by administrators and stewards and matching rules.
+**DataHub Compliance Forms** streamline the process of documenting, annotating, and classifying your most critical Data Assets through a collaborative, crowdsourced approach.
 
-Learn more about forms in the [Documentation Forms Feature Guide](../../../docs/features/feature-guides/documentation-forms.md).
+With Compliance Forms, you can execute large-scale compliance initiatives by assigning tasks (e.g., documentation, tagging, or classification requirements) to the appropriate stakeholders — data owners, stewards, and subject matter experts.
+
+Learn more about forms in the [Compliance Forms Feature Guide](../../../docs/features/feature-guides/compliance-forms/overview.md).
 
 ### Goal Of This Guide
 This guide will show you how to 
diff --git a/docs/api/tutorials/structured-properties.md b/docs/api/tutorials/structured-properties.md
index 00e992f2bd0bb..2caa015e20659 100644
--- a/docs/api/tutorials/structured-properties.md
+++ b/docs/api/tutorials/structured-properties.md
@@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
  Structured properties are a structured, named set of properties that can be attached to logical entities like Datasets, DataJobs, etc.
 Structured properties have values that are types. Conceptually, they are like “field definitions”.
 
-Learn more about structured properties in the [Structured Properties Feature Guide](../../../docs/features/feature-guides/properties.md).
+Learn more about structured properties in the [Structured Properties Feature Guide](../../../docs/features/feature-guides/properties/overview.md).
 
 
 ### Goal Of This Guide
@@ -66,14 +66,14 @@ mutation createStructuredProperty {
       qualifiedName:"retentionTime",
       displayName: "Retention Time",
       description: "Retention Time is used to figure out how long to retain records in a dataset",
-      valueType: "urn:li:dataType:number",
+      valueType: "urn:li:dataType:datahub.number",
       allowedValues: [
         {numberValue: 30, description: "30 days, usually reserved for datasets that are ephemeral and contain pii"},
         {numberValue: 90, description:"description: Use this for datasets that drive monthly reporting but contain pii"},
         {numberValue: 365, description:"Use this for non-sensitive data that can be retained for longer"}
       ],
       cardinality: SINGLE,
-      entityTypes: ["urn:li:entityType:dataset", "urn:li:entityType:dataFlow"],
+      entityTypes: ["urn:li:entityType:datahub.dataset", "urn:li:entityType:datahub.dataFlow"],
     }
   ) {
     urn
@@ -532,6 +532,50 @@ Or you can run the following command to view the properties associated with the
 datahub dataset get --urn {urn}
 ```
 
+## Read Structured Properties From a Dataset
+
+For reading all structured properties from a dataset:
+
+<Tabs>
+<TabItem value="graphql" label="GraphQL" default>
+
+```graphql
+query getDataset {
+  dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.customer,PROD)") {
+    structuredProperties {
+      properties {
+        structuredProperty {
+          urn
+          type
+          definition {
+            displayName
+            description
+            allowedValues {
+              description
+            }
+          }
+        }
+        values {
+          ... on StringValue {
+            stringValue
+          }
+          ... on NumberValue {
+            numberValue
+          }
+        }
+        valueEntities {
+          urn
+          type
+        }
+      }
+    }
+  }
+}
+```
+
+</TabItem>  
+</Tabs>
+
 ## Remove Structured Properties From a Dataset
 
 For removing a structured property or list of structured properties from a dataset:
@@ -1733,4 +1777,4 @@ Example Response:
 ```
 
 </TabItem>
-</Tabs>
\ No newline at end of file
+</Tabs>
diff --git a/docs/authentication/guides/sso/configure-oidc-react.md b/docs/authentication/guides/sso/configure-oidc-react.md
index 4dd882cb9a864..0f2c2f5d013bf 100644
--- a/docs/authentication/guides/sso/configure-oidc-react.md
+++ b/docs/authentication/guides/sso/configure-oidc-react.md
@@ -1,202 +1,21 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# OIDC Authentication
+# Configuring OIDC Authentication
 
 The DataHub React application supports OIDC authentication built on top of the [Pac4j Play](https://github.com/pac4j/play-pac4j) library.
 This enables operators of DataHub to integrate with 3rd party identity providers like Okta, Google, Keycloak, & more to authenticate their users.
 
-## 1. Register an app with your Identity Provider
+## 1. Get your credentials
 
-<Tabs>
-<TabItem value="google" label="Google Identity">
+Before you do anything, you'll want to set up DataHub with your SSO provider, and get prerequisite credentials:
+1. _Client ID_ - A unique identifier for your application with the identity provider
+2. _Client Secret_ - A shared secret to use for exchange between you and your identity provider.
+3. _Discovery URL_ - A URL where the OIDC API of your identity provider can be discovered.
 
-#### Create a project in the Google API Console
+[👉 See the steps here](./initialize-oidc.md)
 
-Using an account linked to your organization, navigate to the [Google API Console](https://console.developers.google.com/) and select **New project**.
-Within this project, we will configure the OAuth2.0 screen and credentials.
-
-#### Create OAuth2.0 consent screen
-
-Navigate to **OAuth consent screen**. This is where you'll configure the screen your users see when attempting to
-log in to DataHub. Select **Internal** (if you only want your company users to have access) and then click **Create**.
-Note that in order to complete this step you should be logged into a Google account associated with your organization.
-
-Fill out the details in the App Information & Domain sections. Make sure the 'Application Home Page' provided matches where DataHub is deployed
-at your organization. Once you've completed this, **Save & Continue**.
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google-setup-1.png"/>
-</p>
-
-#### Configure the scopes
-
-Next, click **Add or Remove Scopes**. Select the following scope and click **Save & Continue**.
-
-- .../auth/userinfo.email
-- .../auth/userinfo.profile
-- openid
-
-</TabItem>
-<TabItem value="okta" label="Okta">
-
-#### Create an application in Okta Developer Console
-
-Log in to your Okta admin account & navigate to the developer console. Select **Applications**, then **Add Application**, the **Create New App** to create a new app.
-Select `Web` as the **Platform**, and `OpenID Connect` as the **Sign on method**.
-
-Click **Create** and name your application under **General Settings** and save.
-
-- **Login Redirect URI** : `https://your-datahub-domain.com/callback/oidc`.
-- **Logout Redirect URI**. `https://your-datahub-domain.com/login`
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-2.png"/>
-</p>
-
-:::note Optional
-If you're enabling DataHub login as an Okta tile, you'll need to provide the **Initiate Login URI**. You
-can set if to `https://your-datahub-domain.com/authenticate`. If you're just testing locally, this can be `http://localhost:9002`.
-:::
-
-</TabItem>
-<TabItem value="azure" label="Azure">
-
-#### Create an application registration in Microsoft Azure portal
-
-Using an account linked to your organization, navigate to the [Microsoft Azure Portal](https://portal.azure.com). Select **App registrations**, then **New registration** to register a new app.
-
-Name your app registration and choose who can access your application.
-
-- **Redirect URI** : Select **Web** as type and enter `https://your-datahub-domain.com/callback/oidc`
-
-Azure supports more than one redirect URI, so both can be configured at the same time from the **Authentication** tab once the registration is complete.
-At this point, your app registration should look like the following. Finally, click **Register**.
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-app-registration.png"/>
-</p>
-
-:::note Optional
-Once registration is done, you will land on the app registration **Overview** tab.  
-On the left-side navigation bar, click on **Authentication** under **Manage** and add extra redirect URIs if need be (if you want to support both local testing and Azure deployments).
-
-For logout URI:
-- **Front-channel logout URL**. `https://your-datahub-domain.com/login`
-
-Finally, click **Save**.
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-authentication.png"/>
-</p>
-
-:::
-
-#### Configure Certificates & secrets
-
-On the left-side navigation bar, click on **Certificates & secrets** under **Manage**.  
-Select **Client secrets**, then **New client secret**. Type in a meaningful description for your secret and select an expiry. Click the **Add** button when you are done.
-Copy the value of your newly create secret since Azure will never display its value afterwards.
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-certificates-secrets.png"/>
-</p>
-
-#### Configure API permissions
-
-On the left-side navigation bar, click on **API permissions** under **Manage**. DataHub requires the following four Microsoft Graph APIs:
-
-- User.Read _(should be already configured)_
-- profile
-- email
-- openid
-
-Click on **Add a permission**, then from the **Microsoft APIs** tab select **Microsoft Graph**, then **Delegated permissions**. From the **OpenId permissions** category, select `email`, `openid`, `profile` and click **Add permissions**.
-
-At this point, you should be looking at a screen like the following:
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-api-permissions.png"/>
-</p>
-
-</TabItem>
-</Tabs>
-
-## 2. Obtain Client Credentials & Discovery URL
-
-The goal of this step should be to obtain the following values, which will need to be configured before deploying DataHub:
-
-- **Client ID** - A unique identifier for your application with the identity provider
-- **Client Secret** - A shared secret to use for exchange between you and your identity provider
-- **Discovery URL** - A URL where the OIDC API of your identity provider can be discovered. This should suffixed by
-  `.well-known/openid-configuration`. Sometimes, identity providers will not explicitly include this URL in their setup guides, though
-  this endpoint _will_ exist as per the OIDC specification. For more info see http://openid.net/specs/openid-connect-discovery-1_0.html.
-
-<Tabs>
-
-<TabItem value="google" label="Google Identity">
-
-**Obtain Client Credentials**
-
-Navigate to the **Credentials** tab. Click **Create Credentials** & select **OAuth client ID** as the credential type.
-
-On the following screen, select **Web application** as your Application Type.
-Add the domain where DataHub is hosted to your 'Authorized Javascript Origins'.
-
-```
-https://your-datahub-domain.com
-```
-
-Add the domain where DataHub is hosted with the path `/callback/oidc` appended to 'Authorized Redirect URLs'. Finally, click **Create**
-
-```
-https://your-datahub-domain.com/callback/oidc
-```
-
-You will now receive a pair of values, a client id and a client secret. Bookmark these for the next step.
-
-</TabItem>
-<TabItem value="okta" label="Okta">
-
-**Obtain Client Credentials**
-
-After registering the app, you should see the client credentials. Bookmark the `Client id` and `Client secret` for the next step.
-
-**Obtain Discovery URI**
-
-On the same page, you should see an `Okta Domain`. Your OIDC discovery URI will be formatted as follows:
-
-```
-https://your-okta-domain.com/.well-known/openid-configuration
-```
-
-For example, `https://dev-33231928.okta.com/.well-known/openid-configuration`.
-
-At this point, you should be looking at a screen like the following:
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-1.png"/>
-</p>
-
-</TabItem>
-<TabItem value="azure" label="Azure">
-
-**Obtain Application (Client) ID**
-
-On the left-side navigation bar, go back to the **Overview** tab. You should see the `Application (client) ID`. Save its value for the next step.
-
-**Obtain Discovery URI**
-
-On the same page, you should see a `Directory (tenant) ID`. Your OIDC discovery URI will be formatted as follows:
-
-```
-https://login.microsoftonline.com/{tenant ID}/v2.0/.well-known/openid-configuration
-```
-
-</TabItem>
-</Tabs>
-
-## 3. Configure DataHub Frontend Server
+## 2. Configure DataHub Frontend Server
 
 ### Docker
 
@@ -314,7 +133,7 @@ AUTH_OIDC_GROUPS_CLAIM=<your-groups-claim-name>
 | AUTH_OIDC_EXTRACT_GROUPS_ENABLED    | Only applies if `AUTH_OIDC_JIT_PROVISIONING_ENABLED` is set to true. This determines whether we should attempt to extract a list of group names from a particular claim in the OIDC attributes. Note that if this is enabled, each login will re-sync group membership with the groups in your Identity Provider, clearing the group membership that has been assigned through the DataHub UI. Enable with care! | false   |
 | AUTH_OIDC_GROUPS_CLAIM              | Only applies if `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` is set to true. This determines which OIDC claims will contain a list of string group names. Accepts multiple claim names with comma-separated values. I.e: `groups, teams, departments`.                                                                                                                                                                     | groups  |
 
-## 4. Restart datahub-frontend-react
+## 3. Restart datahub-frontend-react
 
 Once configured, restarting the `datahub-frontend-react` container will enable an indirect authentication flow in which DataHub delegates authentication to the specified identity provider.
 
diff --git a/docs/authentication/guides/sso/initialize-oidc.md b/docs/authentication/guides/sso/initialize-oidc.md
new file mode 100644
index 0000000000000..1c385c096d841
--- /dev/null
+++ b/docs/authentication/guides/sso/initialize-oidc.md
@@ -0,0 +1,238 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Prerequisites for OIDC Authentication
+
+This guide will walk you through the following steps with your identity provider:
+
+1. Create and register an application with your identity provider.
+2. Obtain client credentials and discovery URI to be used in DataHub.
+
+Choose your identity provider to get started:
+
+<Tabs>
+
+<TabItem value="google" label="Google Identity">
+
+### Step 1. Create and Register your App
+
+#### 1. Create a project in the Google API Console
+
+Using an account linked to your organization, navigate to the [Google API Console](https://console.developers.google.com/) and select **New project**.
+
+Within this project, we will configure the OAuth2.0 screen and credentials.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google/google-setup-1-create-project.png"/>
+</p>
+
+
+#### 2. Create OAuth2.0 consent screen
+
+Navigate to **OAuth consent screen**. This is where you'll configure the screen your users see when attempting to log in to DataHub. Select **Internal** (if you only want your company users to have access) and then click **Create**.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google/google-setup-3-oauth-consent-step1.png"/>
+</p>
+
+_Note that in order to complete this step you should be logged into a Google account associated with your organization._
+
+Fill out the details in the App Information & Domain sections. Make sure the 'Application Home Page' provided matches where DataHub is deployed at your organization.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google/google-setup-3-oauth-consent-step2.png"/>
+</p>
+
+Once you've completed this, **Save & Continue**.
+
+#### 3. Configure the appropriate scopes
+
+Next, click **Add or Remove Scopes**. Select the following scopes and click **Save & Continue**.
+
+- `.../auth/userinfo.email`
+- `.../auth/userinfo.profile`
+- `openid`
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google/google-setup-3-oauth-consent-step3.png"/>
+</p>
+
+### Step 2. Create Client Credentials
+
+The following steps will walk you through generating a Client ID and Client Secret.
+
+1. Navigate to the **Credentials** tab and click **Create Credentials**.
+2. Select **OAuth client ID** as the credential type.
+3. On the next screen, select **Web application** as your Application Type.
+4. In **Authorized JavaScript Origins**, add the domain where you are hosting DataHub, i.e. `https://your-datahub-domain.com`.
+5. In **Authorized Redirect URLs**, add the domain where you are hosting DataHub with the path `/callback/oidc` appended, i.e. `https://your-datahub-domain.com/callback/oidc`.
+6. Click **Create**.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google/google-setup-4-oauth-client-id-2.png"/>
+</p>
+
+This will generate a **Client ID** and **Client Secret**:
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google/google-setup-4-oauth-client-id-3.png"/>
+</p>
+
+You will need these values in the next step, in addition to the following **Discovery URI**:
+
+```
+https://accounts.google.com/.well-known/openid-configuration`
+```
+
+</TabItem>
+
+
+<TabItem value="okta" label="Okta Identity">
+
+### Step 1. Create and Register your App
+
+#### 1. Create an application in Okta Developer Console
+
+Log in to your Okta admin account and navigate to the developer console. From there:
+
+1. Select **Applications**
+2. Click **Add Application**
+3. Click **Create New App**
+4. Select **OpenID Connect** as the Sign On method
+5. Choose **Web** as the Platform
+6. Click **Create**
+
+#### 2. Configure application settings
+
+Under **General Settings**, provide a name for your application and configure the following URIs:
+
+- Login Redirect URI: `https://your-datahub-domain.com/callback/oidc`
+- Logout Redirect URI: `https://your-datahub-domain.com/login`
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-2.png"/>
+</p>
+
+#### 3. Configure Okta Tile (Optional)
+
+If you plan to enable DataHub login as an Okta tile, configure the Initiate Login URI:
+
+- For production: `https://your-datahub-domain.com/authenticate`
+- For local testing: `http://localhost:9002`
+
+### Step 2. Locate Client Credentials and Discovery URI
+
+After registering your app, navigate to the **General** tab to find the following Client Credential values:
+
+* **Client ID**: Public identifier for the client that is required for all OAuth flows.
+* **Client Secret**: Secret used by the client to exchange an authorization code for a token.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-1.png"/>
+</p>
+
+You will need these values in the next step, in addition to the following **Discovery URI**:
+
+```
+https://your-okta-domain.com/.well-known/openid-configuration
+```
+
+</TabItem>
+
+
+<TabItem value="azure" label="Azure AD">
+
+### Step 1. Create and Register your App
+
+#### 1. Create an application in Microsoft Azure portal
+
+Using an account linked to your organization, navigate to the [Microsoft Azure Portal](https://portal.azure.com). From there:
+
+1. Select **App Registrations**.
+2. Click **New Registration** to register a new app.
+3. Provide a Name for the application and choose the supported account types.
+4. Under **Redirect URI**, choose **Web** and enter `https://your-datahub-domain.com/callback/oidc`. NOTE: You can add more later.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-app-registration.png"/>
+</p>
+
+5. Click **Register**.
+
+#### 2. Configure Logout URL
+
+Once registration is complete, you will need to configure the Logout URL, which is required for SSO to work correctly.
+
+1. Navigate to **Authentication** from the left-side navigation menu.
+2. Set **Front-channel logout URL** to `https://your-datahub-domain.com/login`.
+3. Optionally add additional Redirect URIs, such as `http://localhost:9002/callback/oidc` for local testing.
+4. Click **Save**.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-authentication.png"/>
+</p>
+
+
+### Step 2. Client Credentials and Discovery URI
+
+#### 1. Generate a Client Secret
+
+You are now ready to create and configure client credentials:
+
+1. Click **Certificates & secrets** from the left-side navigation menu.
+2. Select **Client secrets**, then **New client secret**.
+3. Provide a Name for the secret and set an expiry.
+4. Click **Add**.
+5. Copy the secret **`Value`** to be used as the **Client Secret** in DataHub SSO configuration; **Azure will not display this again**.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-certificates-secrets.png"/>
+</p>
+
+#### 2. Configure API Permissions
+
+Next, you will configure the appropriate API permissions to enable SSO with DataHub.
+
+1. Click **API permissions** from the left-side navigation menu.
+2. Click **Add a permission**.
+3. Under the **Microsoft APIs** tab, select **Microsoft Graph**, then **Delegated permissions**.
+4. Under the **OpenId permissions** category, select the following:
+  
+  - `User.Read`
+  - `profile`
+  - `email`
+  - `openid`
+
+5. Click **Add permissions**.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-api-permissions.png"/>
+</p>
+
+#### 3. Locate Client Credentials and Discovery URI
+
+Now that you have registered your app, generated a client secret, and configured the appropriate permissions, you are now ready to enable Azure AD SSO with DataHub. 
+
+You will need the following values in the next step:
+
+- **Application (client) ID**: Find this on the **Overview** tab. This will map to **Client ID** in DataHub.
+- **Client Secret**: Stored as `Value` in the Client secret you created, above. This will map to **Client Secret** in DataHub.
+- **Directory (tenant) ID**: Located on the **Overview** tab. This will map to **Discovery URI** in DataHub. It will be formatted as `https://login.microsoftonline.com/{tenant ID}/v2.0/.well-known/openid-configuration`.
+
+<p align="center">
+  <img width="80%"  src="https://learn.microsoft.com/en-us/azure/active-directory-b2c/media/client-credentials-grant-flow/get-application-id.png"/>
+</p>
+
+</TabItem>
+</Tabs>
+
+### Next Steps
+
+Once you have your **Client ID**, **Client Secret**, and **Discovery URI**, you may proceed with next steps.
+
+### DataHub Cloud
+If you're deployed with DataHub Cloud, you can enable OIDC SSO with a few clicks. [👉 See the guide here](../../../managed-datahub/integrations/oidc-sso-integration.md).
+
+### Open Source
+If you're self-deployed with DataHub Core, you'll need to configure your frontend server within your deployment environment. [👉 See the guide here](./configure-oidc-react.md).
+
diff --git a/docs/automations/ai-docs.md b/docs/automations/ai-docs.md
new file mode 100644
index 0000000000000..bbec33f3bcae6
--- /dev/null
+++ b/docs/automations/ai-docs.md
@@ -0,0 +1,36 @@
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# AI Documentation
+
+<FeatureAvailability saasOnly />
+
+:::info
+
+This feature is currently in closed beta. Reach out to your Acryl representative to get access.
+
+:::
+
+With AI-powered documentation, you can automatically generate documentation for tables and columns.
+
+<p align="center">
+<iframe width="560" height="315" src="https://www.youtube.com/embed/_7DieZeZspY?si=Q5FkCA0gZPEFMj0Y" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+</p>
+
+## Configuring
+
+No configuration is required - just hit "Generate" on any table or column in the UI.
+
+## How it works
+
+Generating good documentation requires a holistic understanding of the data. Information we take into account includes, but is not limited to:
+
+- Dataset name and any existing documentation
+- Column name, type, description, and sample values
+- Lineage relationships to upstream and downstream assets
+- Metadata about other related assets
+
+Data privacy: Your metadata is not sent to any third-party LLMs. We use AWS Bedrock internally, which means all metadata remains within the Acryl AWS account. We do not fine-tune on customer data.
+
+## Limitations
+
+- This feature is powered by an LLM, which can produce inaccurate results. While we've taken steps to reduce the likelihood of hallucinations, they can still occur.
diff --git a/docs/automations/ai-term-suggestion.md b/docs/automations/ai-term-suggestion.md
new file mode 100644
index 0000000000000..27d1716cfc372
--- /dev/null
+++ b/docs/automations/ai-term-suggestion.md
@@ -0,0 +1,72 @@
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# AI Glossary Term Suggestions
+
+<FeatureAvailability saasOnly />
+
+:::info
+
+This feature is currently in closed beta. Reach out to your Acryl representative to get access.
+
+:::
+
+The AI Glossary Term Suggestion automation uses LLMs to suggest [Glossary Terms](../glossary/business-glossary.md) for tables and columns in your data.
+
+This is useful for improving coverage of glossary terms across your organization, which is important for compliance and governance efforts.
+
+This automation can:
+
+- Automatically suggests glossary terms for tables and columns.
+- Goes beyond a predefined set of terms and works with your business glossary.
+- Generates [proposals](../managed-datahub/approval-workflows.md) for owners to review, or can automatically add terms to tables/columns.
+- Automatically adjusts to human-provided feedback and curation (coming soon).
+
+## Prerequisites
+
+- A business glossary with terms defined. Additional metadata, like documentation and existing term assignments, will improve the accuracy of our suggestions.
+
+## Configuring
+
+1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar.
+
+  <p align="center">
+    <img width="30%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automations-nav-link.png"/>
+  </p>
+
+2. **Create the Automation**: Click on 'Create' and select 'AI Glossary Term Suggestions'.
+
+  <p align="center">
+    <img width="40%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/ai-term-suggestion/automation-type.png"/>
+  </p>
+
+3. **Configure the Automation**: Fill in the required fields to configure the automation.
+   The main fields to configure are (1) what terms to use for suggestions and (2) what entities to generate suggestions for.
+
+  <p align="center">
+    <img width="50%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/ai-term-suggestion/automation-config.png"/>
+  </p>
+
+4. Once it's enabled, that's it! You'll start to see terms show up in the UI, either on assets or in the proposals page.
+
+  <p align="center">
+    <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/ai-term-suggestion/term-proposals.png"/>
+  </p>
+
+## How it works
+
+The automation will scan through all the datasets matched by the configured filters. For each one, it will generate suggestions.
+If new entities are added that match the configured filters, those will also be classified within 24 hours.
+
+We take into account the following metadata when generating suggestions:
+
+- Dataset name and description
+- Column name, type, description, and sample values
+- Glossary term name, documentation, and hierarchy
+- Feedback loop: existing assignments and accepted/rejected proposals (coming soon)
+
+Data privacy: Your metadata is not sent to any third-party LLMs. We use AWS Bedrock internally, which means all metadata remains within the Acryl AWS account. We do not fine-tune on customer data.
+
+## Limitations
+
+- A single configured automation can classify at most 10k entities.
+- We cannot do partial reclassification. If you add a new column to an existing table, we won't regenerate suggestions for that table.
diff --git a/docs/automations/bigquery-metadata-sync.md b/docs/automations/bigquery-metadata-sync.md
new file mode 100644
index 0000000000000..3d896f57d341a
--- /dev/null
+++ b/docs/automations/bigquery-metadata-sync.md
@@ -0,0 +1,186 @@
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# BigQuery Metadata Sync Automation
+
+<FeatureAvailability saasOnly />
+
+## Introduction
+
+BigQuery Metadata Sync is an automation that synchronizes DataHub Tags, Table and Column descriptions, and Column Glossary Terms with
+BigQuery. This automation is exclusively available in DataHub Cloud (Acryl).
+
+## Use-Cases
+
+- Maintain consistent metadata across DataHub and BigQuery
+- Improve data discovery by propagating rich descriptions back to BigQuery
+- Enhance data governance by applying Policy Tags based on DataHub Glossary Terms
+- Streamline data classification by syncing DataHub Tags to BigQuery Labels
+- Facilitate compliance efforts by automatically tagging sensitive data columns
+- Support data lineage tracking by keeping metadata aligned across platforms
+
+## Sync Capabilities
+
+| DataHub Source | BigQuery Target | Sync Direction | Notes |
+|----------------|-----------------|----------------|--------|
+| Table Tags | Table Labels | Bi-directional | Changes in either system reflect in both |
+| Table Descriptions | Table Descriptions | Bi-directional | Changes in either system reflect in both |
+| Column Descriptions | Column Descriptions | Bi-directional | Changes in either system reflect in both. <br/> Thes sync doesn't delete table description from BigQuery |
+| Column Glossary Terms | Column Policy Tags | DataHub → BigQuery | Created under DataHub taxonomy |
+
+## Setup Instructions
+
+### 1. Verify Permissions
+
+Ensure your service account has the following permissions:
+
+| Task | Required Permissions | Available Role |
+|------|---------------------|----------------|
+| Policy Tag Management | • `datacatalog.taxonomies.create`<br/>• `datacatalog.taxonomies.update`<br/>• `datacatalog.taxonomies.list`<br/>• `datacatalog.taxonomies.get`<br/>• `bigquery.tables.createTagBinding` | Policy Tag Admin |
+| Policy Tag Assignment | • `bigquery.tables.updateTag` | - |
+| Description Management | • `bigquery.tables.update` | - |
+| Label Management | • `bigquery.tables.update` | - |
+
+**Note**: `bigquery.tables` permissions must be granted in every project where metadata sync is needed.
+
+### 2. Enable the Automation
+
+1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar.
+
+  <p align="left">
+    <img width="30%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automations-nav-link.png"/>
+  </p>
+
+2. **Create An Automation**: Click on 'Create' and select 'BigQuery Tag Propagation'.
+
+  <p align="left">
+    <img width="50%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/bigquery-propagation/automation-type.png"/>
+  </p>
+
+3. **Configure Automation**:
+
+    1. **Select a Propagation Action**
+
+    <p align="left">
+      <img width="50%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/bigquery-propagation/automation-form.png"/>
+    </p>
+
+    | Propagation Type | DataHub Entity | BigQuery Entity | Note |
+    | -------- | ------- | ------- | ------- |
+    | Table Tags as Labels | [Table Tag](https://datahubproject.io/docs/tags/) | [BigQuery Label](https://cloud.google.com/bigquery/docs/labels-intro) | - |
+    | Column Glossary Terms as Policy Tags | [Glossary Term on Table Column](https://datahubproject.io/docs/0.14.0/glossary/business-glossary/) | [Policy Tag](https://cloud.google.com/bigquery/docs/best-practices-policy-tags) | <ul><li>Assigned Policy tags are created under DataHub taxonomy.</li></ul><ul><li>Only the latest assigned glossary term set as policy tag. BigQuery only supports one assigned policy tag.</li></ul> <ul><li>Policy Tags are not synced to DataHub as glossary term from BigQuery.</li></ul>
+    | Table Descriptions | [Table Description](https://datahubproject.io/docs/api/tutorials/descriptions/) | Table Description | - |
+    | Column Descriptions | [Column Description](https://datahubproject.io/docs/api/tutorials/descriptions/) | Column Description | - |
+
+    :::note
+
+    You can limit propagation based on specific Tags and Glossary Terms. If none are selected, ALL Tags or Glossary Terms will be automatically propagated to BigQuery tables and columns. (The recommended approach is to not specify a filter to avoid inconsistent states.)
+
+    :::
+
+    :::note
+
+    - BigQuery supports only one Policy Tag per table field. Consequently, the most recently assigned Glossary Term will be set as the Policy Tag for that field.
+    - Policy Tags cannot be applied to fields in External tables. Therefore, if a Glossary Term is assigned to a field in an External table, it will not be applied.
+
+    :::
+
+    2. **Fill in the required fields to connect to BigQuery, along with the name, description, and category**
+
+    <p align="left">
+      <img width="50%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/bigquery-propagation/connection_config.png"/>
+    </p>
+
+    3. **Finally, click 'Save and Run' to start the automation**
+
+## 3. Propagating for Existing Assets (Optional)
+
+To ensure that all existing table Tags and Column Glossary Terms are propagated to BigQuery, you can back-fill historical data for existing assets. Note that the initial back-filling process may take some time, depending on the number of BigQuery assets you have.
+
+To do so, follow these steps:
+
+1. Navigate to the Automation you created in Step 3 above
+2. Click the 3-dot "More" menu
+
+<p align="left">
+  <img width="30%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-more-menu.png"/>
+</p>
+
+3. Click "Initialize"
+
+<p align="left">
+  <img width="50%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-initialize.png"/>
+</p>
+
+This one-time step will kick off the back-filling process for existing descriptions. If you only want to begin propagating descriptions going forward, you can skip this step.
+
+## Viewing Propagated Tags
+
+You can view propagated Tags inside the BigQuery UI to confirm the automation is working as expected.
+
+<p align="left">
+  <img width="50%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/bigquery-propagation/labels.png"/>
+</p>
+
+## Troubleshooting BigQuery Propagation
+
+### Q: What metadata elements support bi-directional syncing between DataHub and BigQuery?
+
+A: The following metadata elements support bi-directional syncing:
+
+- Tags (via BigQuery Labels): Changes made in either DataHub Table Tags or BigQuery Table Labels will be reflected in the other system.
+- Descriptions: Both table and column descriptions are synced bi-directionally.
+
+### Q: Are Policy Tags bi-directionally synced?
+
+A: No, BigQuery Policy Tags are only propagated from DataHub to BigQuery, not vice versa. This means that Policy Tags should be mastered in DataHub using the [Business Glossary](https://datahubproject.io/docs/glossary/business-glossary/).
+
+It is recommended to avoid enabling `extract_policy_tags_from_catalog` during
+ingestion, as this will ingest policy tags as BigQuery labels. Our sync process
+propagates Glossary Term assignments to BigQuery as Policy Tags.
+
+In a future release, we plan to remove this restriction to support full bi-directional syncing.
+
+### Q: What metadata is synced from BigQuery to DataHub during ingestion?
+
+A: During ingestion from BigQuery:
+
+- Tags and descriptions from BigQuery will be ingested into DataHub.
+- Existing Policy Tags in BigQuery will not overwrite or create Business Glossary Terms in DataHub. It only syncs assigned column Glossary Terms from DataHub to BigQuery.
+
+### Q: Where should I manage my Business Glossary?
+
+A: The expectation is that you author and manage the glossary in DataHub. Policy tags in BigQuery should be treated as a reflection of the DataHub glossary, not as the primary source of truth.
+
+### Q: Are there any limitations with Policy Tags in BigQuery?
+
+A: Yes, BigQuery only supports one Policy Tag per column. If multiple glossary
+terms are assigned to a column in DataHub, only the most recently assigned term
+will be set as the policy tag in BigQuery. To reduce the scope of conflicts, you
+can set up filters in the BigQuery Metadata Sync to only synchronize terms from
+a specific area of the Business Glossary.
+
+### Q: How frequently are changes synced between DataHub and BigQuery?
+
+A: From DataHub to BigQuery, the sync happens instantly (within a few seconds)
+when the change occurs in DataHub.
+
+From BigQuery to DataHub, changes are synced when ingestion occurs, and the frequency depends on your custom ingestion schedule. (Visible on the **Integrations** page)
+
+### Q: What happens if there's a conflict between DataHub and BigQuery metadata?
+
+A: In case of conflicts (e.g., a tag is modified in both systems between syncs), the DataHub version will typically take precedence. However, it's best to make changes in one system consistently to avoid potential conflicts.
+
+### Q: What permissions are required for bi-directional syncing?
+
+A: Ensure that the service account used for the automation has the necessary permissions in both DataHub and BigQuery to read and write metadata. See the required BigQuery permissions at the top of the page.
+
+### Q: Can table description removed?
+
+No, the sync can only modify table description but it won't remove or clear a description from a table. 
+
+## Related Documentation
+
+- [DataHub Tags Documentation](https://datahubproject.io/docs/tags/)
+- [DataHub Glossary Documentation](https://datahubproject.io/docs/glossary/business-glossary/)
+- [BigQuery Labels Documentation](https://cloud.google.com/bigquery/docs/labels-intro)
+- [BigQuery Policy Tags Documentation](https://cloud.google.com/bigquery/docs/best-practices-policy-tags)
diff --git a/docs/automations/docs-propagation.md b/docs/automations/docs-propagation.md
index e8eba08d3640e..9f38902894191 100644
--- a/docs/automations/docs-propagation.md
+++ b/docs/automations/docs-propagation.md
@@ -26,7 +26,6 @@ This feature is enabled by default in Open Source DataHub.
 | Column-Level Docs Propagation   | ✔️           | ✔️             |
 | Asset-Level Docs Propagation    | ✔️           | ✔️             |
 | Downstream Lineage + Siblings   | ✔️           | ✔️             |
-| Propagation Rollback (Undo)     | ❌           | ✔️             |
 | Historical Backfilling          | ❌           | ✔️             |
 
 ## Enabling Documentation Propagation
@@ -95,27 +94,6 @@ and then click "Initialize".
 This one-time step will kick off the back-filling process for existing descriptions. If you only want to begin propagating
 descriptions going forward, you can skip this step. 
 
-## Rolling Back Propagated Descriptions (DataHub Cloud Only)
-
-In DataHub Cloud, you can rollback all descriptions that have been propagated historically.
-
-This feature allows you to "clean up" or "undo" any accidental propagation that may have occurred automatically, in the case
-that you no longer want propagated descriptions to be visible. 
-
-To do this, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu
-
-<p align="left">
-  <img width="15%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-more-menu.png"/>
-</p>
-
-and then click "Rollback".
-
-<p align="left">
-  <img width="15%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-rollback.png"/>
-</p>
-
-This one-time step will remove all propagated tags and glossary terms from Snowflake. To simply stop propagating new tags, you can disable the automation.
-
 ## Viewing Propagated Descriptions
 
 Once the automation is enabled, you'll be able to recognize propagated descriptions as those with the thunderbolt icon next to them:
diff --git a/docs/automations/glossary-term-propagation.md b/docs/automations/glossary-term-propagation.md
new file mode 100644
index 0000000000000..90e8e75ea44ef
--- /dev/null
+++ b/docs/automations/glossary-term-propagation.md
@@ -0,0 +1,61 @@
+# Glossary Term Propagation Automation
+
+<FeatureAvailability saasOnly />
+
+## Introduction
+
+Glossary Term Propagation is an automation feature that propagates classification labels (Glossary Terms) across column and assets based on downstream lineage and sibling relationships.
+This automation simplifies metadata management by ensuring consistent term classification and reducing manual effort in categorizing data assets, aiding Data Governance & Compliance, and enhancing Data Discovery.
+
+## Capabilities
+
+- **Column-Level Glossary Term Propagation**: Automatically propagate Glossary Terms to all downstream lineage columns and sibling columns.
+- **Asset-Level Glossary Term Propagation**: Automatically propagate Glossary Terms to all downstream lineage assets & sibling assets.
+- **Select Terms & Term Groups**: Select specific Glossary Terms & Term Groups to propagate, e.g. to propagate only sensitive or important labels. 
+
+Note that Asset-level propagation is currently only support for **Datasets** (Tables, Views, Topics, etc), and not for other asset types including
+Charts, Dashboards, Data Pipelines, Data Tasks.
+
+## Enabling Glossary Term Propagation
+
+1. **Navigate to Automations**: Go to 'Govern' > 'Automations' in the navigation bar.
+
+<p align="center">
+  <img width="20%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automations-nav-link.png"/>
+</p>
+
+
+2. **Create An Automation**: Select 'Glossary Term Propagation' from the automation types.
+
+<p align="center">
+  <img width="30%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/glossary-term-propagation/automation-type.png"/>
+</p>
+
+3. **Configure Automation**: Complete the required fields and select 'Save and Run' to activate the automation.
+
+<p align="center">
+  <img width="60%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/glossary-term-propagation/automation-form.png"/>
+</p>
+
+## Propagating for Existing Assets
+
+In DataHub Cloud, you can back-fill historical data to ensure existing Glossary Terms are consistently propagated across downstream relationships. To begin, access the Automation created in Step 3, click the 3-dot "more" menu, and choose "Initialize." This will kick off the backfill process.
+
+<p align="left">
+  <img width="15%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-more-menu.png"/>
+</p>
+
+and then click "Initialize".
+
+<p align="left">
+  <img width="15%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-initialize.png"/>
+</p>
+
+
+## Viewing Propagated Glossary Terms
+
+Once enabled, propagated Glossary Terms will display a thunderbolt icon, indicating the origin of the term and any intermediate lineage hops used in propagation.
+
+<p align="center">
+  <img width="50%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/glossary-term-propagation/view-propagated-terms.png"/>
+</p>
\ No newline at end of file
diff --git a/docs/automations/snowflake-tag-propagation.md b/docs/automations/snowflake-tag-propagation.md
index bdc80376dfb48..8eded451644cc 100644
--- a/docs/automations/snowflake-tag-propagation.md
+++ b/docs/automations/snowflake-tag-propagation.md
@@ -1,10 +1,11 @@
-
 import FeatureAvailability from '@site/src/components/FeatureAvailability';
 
 # Snowflake Tag Propagation Automation
 
 <FeatureAvailability saasOnly />
 
+> Note that this Automation in currently in open **Beta**. With any questions or issues, please reach out to your Acryl representative. 
+
 ## Introduction
 
 Snowflake Tag Propagation is an automation that allows you to sync DataHub Glossary Terms and Tags on
@@ -16,26 +17,61 @@ both columns and tables back to Snowflake. This automation is available in DataH
 - Automatically Add DataHub Tags to Snowflake Tables and Columns
 - Automatically Remove DataHub Glossary Terms and Tags from Snowflake Tables and Columns when they are removed in DataHub
 
+## Prerequisites
+
+### Permissions Required for Tag Management
+
+- `CREATE TAG`: Required to create new tags in Snowflake.
+Ensure the user or role has this privilege on the specific schema or database where tags will be created.
+- `APPLY TAG`: Required to assign tags to Snowflake objects such as tables, columns, or other database objects.
+This permission must be granted at the database, schema, or object level depending on the scope.
+
+
+### Permissions Required for Object Access
+
+- `USAGE` on the database and schema: Allows access to the database and schema to view and apply changes.
+- `SELECT` on the objects (tables, views, etc.): Enables the automation to read metadata and verify existing tags.
+
+### Example Permission Grant Statements
+
+To grant the necessary permissions for a specific role (DATAHUB_AUTOMATION_ROLE), you can use the following SQL commands:
+
+```sql
+-- Tag management permissions
+GRANT CREATE TAG ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE;
+GRANT APPLY TAG ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE;
+
+-- Object access for metadata operations
+GRANT USAGE ON DATABASE your_database TO ROLE DATAHUB_AUTOMATION_ROLE;
+GRANT USAGE ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE;
+GRANT SELECT ON ALL TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE;
+
+-- Future privileges for tagging
+GRANT SELECT ON FUTURE TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE;
+GRANT APPLY TAG ON FUTURE TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE;
+```
+
+
 ## Enabling Snowflake Tag Sync
 
 1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar.
 
-<p align="left">
-  <img width="20%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automations-nav-link.png"/>
+<p align="center">
+  <img width="20%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automations-nav-link.png"/>
 </p>
 
 2. **Create An Automation**: Click on 'Create' and select 'Snowflake Tag Propagation'.
 
-<p align="left">
-  <img width="30%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/snowflake-tag-propagation/automation-type.png"/>
+<p align="center">
+  <img width="60%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/snowflake-tag-propagation/automation-type.png"/>
 </p>
 
-3. **Configure Automation**: Fill in the required fields to connect to Snowflake, along with the name, description, and category. 
-Note that you can limit propagation based on specific Tags and Glossary Terms. If none are selected, then ALL Tags or Glossary Terms will be automatically
-propagated to Snowflake tables and columns. Finally, click 'Save and Run' to start the automation
+3. **Configure Automation**: Fill in the required fields to connect to Snowflake, along with the name, description, and category.
+   Note that you can limit propagation based on specific Tags and Glossary Terms. If none are selected, then ALL Tags or Glossary Terms will be automatically
+   propagated to Snowflake tables and columns. Finally, click 'Save and Run' to start the automation
 
-<p align="left">
-  <img width="30%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/snowflake-tag-propagation/automation-form.png"/>
+<p align="center">
+  <img width="60%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/snowflake-tag-propagation/automation-form.png"/>
 </p>
 
 ## Propagating for Existing Assets
@@ -46,43 +82,22 @@ Note that it may take some time to complete the initial back-filling process, de
 To do so, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu
 
 <p align="left">
-  <img width="15%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-more-menu.png"/>
+  <img width="20%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-more-menu.png"/>
 </p>
 
 and then click "Initialize".
 
 <p align="left">
-  <img width="15%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-initialize.png"/>
+  <img width="20%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-initialize.png"/>
 </p>
 
 This one-time step will kick off the back-filling process for existing descriptions. If you only want to begin propagating
 descriptions going forward, you can skip this step.
 
-## Rolling Back Propagated Tags
-
-You can rollback all tags and glossary terms that have been propagated historically.
-
-This feature allows you to "clean up" or "undo" any accidental propagation that may have occurred automatically, in the case
-that you no longer want propagated descriptions to be visible.
-
-To do this, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu
-
-<p align="left">
-  <img width="15%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-more-menu.png"/>
-</p>
-
-and then click "Rollback".
-
-<p align="left">
-  <img width="15%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/automation-rollback.png"/>
-</p>
-
-This one-time step will remove all propagated tags and glossary terms from Snowflake. To simply stop propagating new tags, you can disable the automation.
-
 ## Viewing Propagated Tags
 
-You can view propagated Tags (and corresponding DataHub URNs) inside the Snowflake UI to confirm the automation is working as expected. 
+You can view propagated Tags (and corresponding DataHub URNs) inside the Snowflake UI to confirm the automation is working as expected.
 
-<p align="left">
-  <img width="50%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/snowflake-tag-propagation/view-snowflake-tags.png"/>
+<p align="center">
+  <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/automation/saas/snowflake-tag-propagation/view-snowflake-tags.png"/>
 </p>
diff --git a/docs/businessattributes.md b/docs/businessattributes.md
index 1744f48f879e8..3e912e7e60980 100644
--- a/docs/businessattributes.md
+++ b/docs/businessattributes.md
@@ -28,7 +28,6 @@ Taking the example of "United States- Social Security Number", if an application
 What you need to create/update and associate business attributes to dataset schema field
 
 * **Manage Business Attributes** platform privilege to create/update/delete business attributes.
-* **Edit Dataset Column Business Attribute** metadata privilege to associate business attributes to dataset schema field.
 
 ## Using Business Attributes
 As of now Business Attributes can only be created through UI
diff --git a/docs/cli.md b/docs/cli.md
index c109d02e0ad51..1c38077d0d12e 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -115,6 +115,19 @@ datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --dry-run
 datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n
 ```
 
+#### ingest --list-source-runs
+
+The `--list-source-runs` option of the `ingest` command lists the previous runs, displaying their run ID, source name, 
+start time, status, and source URN. This command allows you to filter results using the --urn option for URN-based 
+filtering or the --source option to filter by source name (partial or complete matches are supported).
+
+```shell
+# List all ingestion runs
+datahub ingest --list-source-runs
+# Filter runs by a source name containing "demo"
+datahub ingest --list-source-runs --source "demo"
+```
+
 #### ingest --preview
 
 The `--preview` option of the `ingest` command performs all of the ingestion steps, but limits the processing to only the first 10 workunits produced by the source.
@@ -705,6 +718,7 @@ Please see our [Integrations page](https://datahubproject.io/integrations) if yo
 | [datahub-lineage-file](./generated/ingestion/sources/file-based-lineage.md)                    | _no additional dependencies_                               | Lineage File source                     |
 | [datahub-business-glossary](./generated/ingestion/sources/business-glossary.md)                | _no additional dependencies_                               | Business Glossary File source           |
 | [dbt](./generated/ingestion/sources/dbt.md)                                                    | _no additional dependencies_                               | dbt source                              |
+| [dremio](./generated/ingestion/sources/dremio.md)                                              | `pip install 'acryl-datahub[dremio]'`                      | Dremio Source                          |
 | [druid](./generated/ingestion/sources/druid.md)                                                | `pip install 'acryl-datahub[druid]'`                       | Druid Source                            |
 | [feast](./generated/ingestion/sources/feast.md)                                                | `pip install 'acryl-datahub[feast]'`                       | Feast source (0.26.0)                   |
 | [glue](./generated/ingestion/sources/glue.md)                                                  | `pip install 'acryl-datahub[glue]'`                        | AWS Glue source                         |
diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md
index 49b0ea1d69ae1..c625ba26a3865 100644
--- a/docs/deploy/aws.md
+++ b/docs/deploy/aws.md
@@ -53,7 +53,18 @@ ip-192-168-64-56.us-west-2.compute.internal   Ready    <none>   3h    v1.18.9-ek
 ip-192-168-8-126.us-west-2.compute.internal   Ready    <none>   3h    v1.18.9-eks-d1db3c
 ```
 
-Once your cluster is running, make sure to install the EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes. [add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html)
+### Install EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes
+
+Once your cluster is running, make sure to install the EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes. [add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html).  By default Core DNS and VPC CNI plugins are installed.  You need to manually install the EBS CSI driver. It show look this in your console when you are done.
+
+![Screenshot 2024-11-15 at 4 42 09 PM](https://github.com/user-attachments/assets/5a9a2af0-e804-4896-85bb-dc5834208719)
+
+### Add the AmazonEBSCSIDriverPolicy role to the EKS node group 
+
+Next is to add the AmazonEBSCSIDriverPolicy role to the EKS node group.    You will from the EKS Node group by going to the Compute tab in your EKS cluster and clicking on the IAM entry for the EKS node group.  Add the AmazonEBSCSIDriverPolicy policy. 
+
+![Screenshot 2024-11-15 at 4 42 29 PM](https://github.com/user-attachments/assets/8971c8d6-8543-408b-9a07-814aacb2532d)
+![Screenshot 2024-11-15 at 4 42 46 PM](https://github.com/user-attachments/assets/397f9131-5f13-4d9f-a664-9921d9bbf44e)
 
 ## Setup DataHub using Helm
 
diff --git a/docs/deploy/environment-vars.md b/docs/deploy/environment-vars.md
index 6429996c088b4..13d7410397533 100644
--- a/docs/deploy/environment-vars.md
+++ b/docs/deploy/environment-vars.md
@@ -9,12 +9,13 @@ DataHub works.
 
 ## Feature Flags
 
-| Variable                                         | Default | Unit/Type | Components                              | Description                                                                                                                 |
-|--------------------------------------------------|---------|-----------|-----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
-| `UI_INGESTION_ENABLED`                           | `true`  | boolean   | [`GMS`, `MCE Consumer`]                 | Enable UI based ingestion.                                                                                                  |
-| `DATAHUB_ANALYTICS_ENABLED`                      | `true`  | boolean   | [`Frontend`, `GMS`]                     | Collect DataHub usage to populate the analytics dashboard.                                                                  |
-| `BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE` | `true`  | boolean   | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Do not wait for the `system-update` to complete before starting. This should typically only be disabled during development. |
-| `ER_MODEL_RELATIONSHIP_FEATURE_ENABLED`          | `false` | boolean   | [`Frontend`, `GMS`]                     | Enable ER Model Relation Feature that shows Relationships Tab within a Dataset UI.                                          |
+| Variable                                         | Default  | Unit/Type | Components                               | Description                                                                                                                 |
+|--------------------------------------------------|----------|-----------|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+| `UI_INGESTION_ENABLED`                           | `true`   | boolean   | [`GMS`, `MCE Consumer`]                  | Enable UI based ingestion.                                                                                                  |
+| `DATAHUB_ANALYTICS_ENABLED`                      | `true`   | boolean   | [`Frontend`, `GMS`]                      | Collect DataHub usage to populate the analytics dashboard.                                                                  |
+| `BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE` | `true`   | boolean   | [`GMS`, `MCE Consumer`, `MAE Consumer`]  | Do not wait for the `system-update` to complete before starting. This should typically only be disabled during development. |
+| `ER_MODEL_RELATIONSHIP_FEATURE_ENABLED`          | `false`  | boolean   | [`Frontend`, `GMS`]                      | Enable ER Model Relation Feature that shows Relationships Tab within a Dataset UI.                                          |
+| `STRICT_URN_VALIDATION_ENABLED`                  | `false`  | boolean   | [`GMS`, `MCE Consumer`, `MAE Consumer`]  | Enable stricter URN validation logic                                                                                        |
 
 
 ## Ingestion
diff --git a/docs/features/feature-guides/compliance-forms/complete-a-form.md b/docs/features/feature-guides/compliance-forms/complete-a-form.md
new file mode 100644
index 0000000000000..285c722179e4d
--- /dev/null
+++ b/docs/features/feature-guides/compliance-forms/complete-a-form.md
@@ -0,0 +1,177 @@
+---
+title: Complete a Form
+---
+
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# Complete a DataHub Compliance Form
+<FeatureAvailability/>
+
+Once a Compliance Form has been published (see [Create a Compliance Form](create-a-form.md)), Assignees will receive notifications in their Task Center prompting them to complete the Form for each Asset they are responsible for.
+
+This guide provides an example of completing a Compliance Form, covering:
+
+1. Accessing a Form from an Asset Page or the Task Center
+2. Completing a Form for a single Asset or multiple Assets (DataHub Cloud only)
+3. Understanding different Form Question completion states
+
+The example uses the **Governance Initiative 2024**, a Verification Form with 3 Required Questions:
+
+<p align="center">
+  <img
+       width="90%"
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/sample-compliance-form.png" 
+       alt="Sample Compliance Form"/>
+</p>
+
+## Access a Compliance Form
+
+Once you have been assigned to complete a Compliance Form, you will see a **Complete Documentation** or **Complete Verification** option on the right-hand side of an Asset Page:
+
+<p align="center">
+  <img
+       width="70%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-task-from-asset-page.png"
+       alt="Open Compliance Form from Asset Page"/>
+</p>
+
+**DataHub Cloud** users can find all outstanding Compliance Form requests by navigating to the **Task Center**:
+
+<p align="center">
+  <img
+       width="80%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-tasks-from-task-center.png"
+       alt="Open Compliance Form from Task Center"/>
+</p>
+
+## Complete a Form for a Single Asset
+
+When filling out a Compliance Form for a single Asset, you'll see a list of Questions tailored to that Asset, with clear labels showing which ones are required. Here's how it works:
+
+- **Question Details:** Each Question specifies if it's required or optional. Required Questions must be completed to submit the Form.
+- **Pre-Populated Metadata:** If metadata already exists for a Question, it will appear pre-filled. You can confirm the existing value or make updates as needed.
+- **Assignee Contributions:** If another Assignee has already provided a response, their name and the time of submission will be displayed. This gives you visibility into previous input, though you can still update the response.
+
+:::tip
+For Verification Forms, after addressing all required Questions, you'll be prompted to provide final sign-off. This ensures all responses are complete and accurate, marking the Form ready for submission.
+:::
+
+Once you complete all required responses, the sidebar will update with the status of the Asset:
+
+- **Documented**: All required Questions are completed, Verification is not needed
+- **Verified**: All required Questions are completed and Verified
+
+Here's what the **Governance Initiative 2024** Verification Form looks like for `dogs_in_movies` after responding to all Required Questions:
+
+<p align="center">
+  <img
+       width="80%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-task-ready-to-verify.png"
+       alt="Asset Ready to Verify"/>
+</p>
+
+And here's the `dogs_in_movies` sidebar after Verifying all responses:
+
+<p align="center">
+  <img
+       width="70%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-task-asset-verified.png"
+       alt="Asset is Verified"/>
+</p>
+
+### Navigate to the Next Asset
+
+To continue working through the Compliance Forms assigned to you, **use the navigation arrows located in the top-right corner**. These arrows will take you to the next Asset that is still pending Form completion or Verification. Only Assets that require action will appear in this flow, allowing you to focus on the remaining tasks without unnecessary steps.
+
+## Complete a Form Question for Multiple Assets
+
+When you want to provide the same response for a question to multiple assets, you can apply it in bulk by selecting the **By Question** option in the top-right corner. This allows you to navigate through the Form question-by-question and apply the same response to multiple assets.
+
+:::note
+Completing Form Questions for multiple Assets is only supported for DataHub Cloud.
+:::
+
+### Example: Apply a Response in Bulk
+
+Let's look at an example. Imagine we are trying to provide the same answer to a Question for all Assets in a Snowflake schema called `DEMO_DB`. Here's how we'd do it:
+
+1. **Filter Assets**: Filter down to all datasets in the `DEMO_DB` Snowflake schema.
+2. **Set a Response**: For the selected Question, provide a response. In this case, we'll set the Deletion Date to be `2024-12-31`.
+3. **Apply to All Selected Assets**: Use the bulk application feature to apply this response to all filtered Assets.
+
+<p align="center">
+  <img
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-tasks-by-question.png"
+       alt="Apply Response to Multiple Assets"/>
+</p>
+
+After setting the response, toggle through each Question, providing the necessary responses to combinations of Assets.
+
+### Verification for Multiple Assets
+
+For Verification Forms, as you complete Questions, you will see the number of assets eligible for Verification in the top-right corner. This makes it easy to track which Assets have met the requirements.
+
+<p align="center">
+  <img
+       width="70%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-tasks-by-question-ready-to-verify.png"
+       alt="Multiple Assets ready to Verify"/>
+</p>
+
+When you are ready to bulk Verify Assets, you will be prompted to confirm that all responses are complete and accurate before proceeding.
+
+<p align="center">
+  <img
+       width="70%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-tasks-bulk-verify.png"
+       alt="Final Bulk Verification"/>
+</p>
+
+### Switch Between Completion Modes
+
+You can easily toggle between the **Complete By Asset** and **Complete By Question** views as needed, ensuring flexibility while completing and verifying the Compliance Forms.
+
+## Understanding Different Form Question Completion States
+
+When completing a Compliance Form, you may encounter various types of Questions, each with unique completion states based on existing metadata or prior responses from other Assignees. This section highlights examples of various completion states to help you understand how Questions can be answered, confirmed, or updated when completing a Form.
+
+**_1. What is the primary use case for this asset?_**
+
+This required Question is asking the Assignee to provide Documentation on how the Asset should be used. Note that there is no text populated in the description, meaning the Asset does not have any documentation at all.
+
+<p align="center">
+  <img
+       width="90%"
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-task-incomplete-question.png"
+       alt="Sample Compliance Form"/>
+</p>
+
+**_2. When will this asset be deleted?_**
+
+You may notice that this question has a pre-populated value. When metadata has been populated from a source _outside_ of a Form, users will have the option to update and save the value, or, simply **Confirm** that the value is accurate.
+
+<p align="center">
+  <img
+       width="90%"
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-task-confirm-value.png"
+       alt="Sample Compliance Form"/>
+</p>
+
+**_3. Who is the Data Steward of this Asset?_**
+
+Here's an example where a different Form Assignee has already provided an answer through the Compliance Form 3 days ago. All Assignees will still have the option to update the response, but this allows users to see how other Form Assignees have already answered the questions.
+
+<p align="center">
+  <img
+       width="90%"
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/completion/complete-task-question-answered.png"
+       alt="Sample Compliance Form"/>
+</p>
+
+
+## FAQ and Troubleshooting
+
+**Why don’t I see any Compliance Forms in the Task Center or on an Asset Page?**
+
+If you don’t see any Compliance Forms, check with the Form author to ensure your DataHub user account has been assigned to complete a Form for one or more Assets. Forms can be assigned to Asset Owners, specific DataHub Users, or a combination of both.
\ No newline at end of file
diff --git a/docs/features/feature-guides/compliance-forms/create-a-form.md b/docs/features/feature-guides/compliance-forms/create-a-form.md
new file mode 100644
index 0000000000000..a768bb16e4f64
--- /dev/null
+++ b/docs/features/feature-guides/compliance-forms/create-a-form.md
@@ -0,0 +1,186 @@
+---
+title: Create a Form
+---
+
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# Create a DataHub Compliance Form
+<FeatureAvailability/>
+
+This guide will walk you through creating and assigning Compliance Forms, including:
+
+1. Creating a new Compliance Form
+2. Building **Questions** for the Compliance Form
+3. Assigning **Assets** for the Compliance Form
+4. Selecting **Assignees** for the Compliance Form
+5. Publishing a Compliance Form
+
+:::note
+Managing Compliance Forms via the DataHub UI is only available in DataHub Cloud. If you are using DataHub Core, please refer to the [Compliance Forms API Guide](../../../api/tutorials/forms.md).
+:::
+
+### Prerequisites
+
+In order to create, edit, or remove Compliance Forms, you must have the **Manage Compliance Forms** Platform privilege.
+
+### Step 1: Create a new Compliance Form
+
+From the navigation bar, head to **Govern** > **Compliance Forms**. Click **+ Create** to start building your Form.
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/management/list-compliance-forms-before.png"
+       alt="View of all Compliance Forms"/>
+</p>
+
+First up, provide the following details:
+
+1. **Name:** Select a unique and descriptive name for your Compliance Form that clearly communicates its purpose, such as **"PII Certification Q4 2024"**.
+
+    _**Pro Tip:** This name will be displayed to Assignees when they are assigned tasks, so make it clear and detailed to ensure it conveys the intent of the Form effectively._  
+
+2. **Description:** Craft a concise yet informative description that explains the purpose of the Compliance Form. Include key details such as the importance of the initiative, its objectives, and the expected completion timeline. This helps Assignees understand the context and significance of their role in the process.
+
+    _**Example:** "This Compliance Form is designed to ensure all datasets containing PII are reviewed and verified by Q4 2024. Completing this Form is critical for compliance with organizational and regulatory requirements."_
+
+3. **Type:** Specify the collection type for the Form, based on your compliance requirements:  
+    - **Completion:** The Form is considered complete once all required questions are answered for the selected Assets. We recommend this option for basic requirement completion use cases.
+
+    - **Verification:** The Form is considered complete only when all required questions are answered for the selected Assets **and** an Assignee has explicitly "verified" the responses. We recommend this option when final sign-off by Assignees is necessary, ensuring they acknowledge the accuracy and validity of their responses.
+
+4. Next, click **Add Question** to begin building the requirements for your Form.
+
+<p align="center">
+  <img
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/management/create-compliance-form-add-question.png"
+       alt="Create a new Compliance Form"/>
+</p>
+
+### Step 2: Build Questions for your Form
+
+Next, define the Questions for your Compliance Forms. These are used to collect required information about selected assets, and must be completed by an Assignee in order for the Form to be considered complete.
+
+There are 5 different question types to choose from:
+
+* **Ownership:** Request one or more owners to be assigned to selected assets. Optionally restrict responses to a specific set of valid users, groups, and ownership types.
+    * _E.g. Who is responsible for ensuring the accuracy of this Dataset?_
+* **Domain:** Assign a Domain to the Asset, with the option to predefine the set of allowed Domains.
+    * _E.g. Which Domain does this Dashboard belong to? Sales, Marketing, Finance._
+* **Documentation:** Provide Documentation about the Asset and/or Column.
+    * _E.g. What is the primary use case of this Dataset? What caveats should others be aware of?_
+* **Glossary Terms:** Assign one or more Glossary Term to the Asset and/or Column, with the option to predefine the set of allowed Glossary Terms. 
+    * _E.g. What types of personally identifiable information (PII) are included in this Asset? Email, Address, SSN, etc._
+* **Structured Properties:** Apply custom properties to an Asset and/or Column.
+    * _E.g. What date will this Dataset be deprecated and deleted?_
+
+When creating a Question, use a clear and concise Title that is easy for Assignees to understand. In the Description, include additional context or instructions to guide their responses. Both the Title and Description will be visible to Assignees when completing the Form, so make sure to provide any specific hints or details they may need to answer the Question accurately and confidently.
+
+<p align="center">
+  <img
+       width="50%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/management/create-compliance-form-prompt.png"
+       alt="Create a new Compliance Form prompt"/>
+</p>
+
+### Step 3: Assign Assets to your Compliance Form
+
+Now that you have defined the Questions you want Assignees to complete, it's now time to assign the in-scope Assets for this exercise.
+
+In the **Assign Assets** section, you can easily target the specific set of Assets that are relevant for this Form with the following steps:
+
+1. Add a Condition or Group of Conditions
+2. Choose the appropriate filter type, such as:
+    * Asset Type (Dataset, Chart, etc.)
+    * Platform (Snowflake, dbt, etc.)
+    * Domain (Sales, Marketing, Finance, etc.)
+    * Assigned Owners
+    * Assigned Glossary Terms
+3. Decide between **All**, **Any**, or **None** of the filters should apply
+4. Preview the relevant Assets to confirm you have applied the appropriate filters
+
+For example, you can apply filters to focus on all **Snowflake Datasets** that are also associated with the **Finance Domain**. This allows you to break down your compliance initiatives into manageable chunks, so you don't have to go after your entire data ecosystem in one go.
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/management/create-compliance-form-assign-assets.png"
+       alt="Assign assets to a Compliance Form"/>
+</p>
+
+### Step 4: Select Assignees to complete your Compliance Form
+
+With the Questions and assigned Assets defined, the next step is to select the Assignees—the Users and/or Groups responsible for completing the Form.
+
+In the **Add Recipients** section, decide who is responsible for completing the Form:
+
+* **Asset Owners:** Any User that is assigned to one of the in-scope Assets will be able to complete the Form. This is useful for larger initiatives when you may not know the full set of Users.
+* **Specific Users and/or Groups:** Select a specific set of Users and/or Groups within DataHub. This is useful when Ownership of the Assets may be poorly-defined.
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/management/create-compliance-form-add-users-or-groups.png"
+       alt="Assign recipients to a Compliance Form"/>
+</p>
+
+### Step 5: Publish your Form
+
+After defining the Questions, assigning Assets, and selecting the Assignees, your Form is ready to be published. Once published, Assignees will be notified to complete the Form for the Assets they are responsible for.
+
+
+To publish a Form, simply click **Publish**.
+
+:::caution
+Once you have published a Form, you **cannot** change or add Questions. You can, however, change the set of Assets and/or Assignees for the Form.
+:::
+
+Not ready for primetime just yet? No worries! You also have the option to **Save Draft**.
+
+<p align="center">
+  <img
+       width="80%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/management/create-compliance-form-publish.png"
+       alt="Publish a Compliance Form"/>
+</p>
+
+## FAQ and Troubleshooting
+
+**Does answering a Compliance Form Question update the selected Asset?**
+
+Yes! Compliance Forms serve as a powerful tool for gathering and updating key attributes for your mission-critical Data Assets at scale. When a Question is answered, the response directly updates the corresponding attributes of the selected Asset.
+
+**How does a Compliance Form interact with existing metadata?**
+
+If an Asset already has existing metadata that is also referenced in a Form Question, Assignees will have the option to confirm the existing value, overwrite the value, or append additional details.
+
+_You can find more details and examples in the [Complete a Form](complete-a-form.md#understanding-different-form-question-completion-states) guide._
+
+**What is the difference between Completion and Verification Forms?**
+
+Both Form types are a way to configure a set of optional and/or required Questions for DataHub users to complete. When using Verification Forms, users will be presented with a final verification step once all required questions have been completed; you can think of this as a final acknowledgment of the accuracy of information submitted.
+
+**Can I assign multiple Forms to a single Asset?**
+
+You sure can! Please keep in mind that an Asset will only be considered Documented or Verified if all required questions are completed on all assigned Forms.
+
+**How will DataHub Users know that a Compliance Form has been assigned to them?**
+
+They have to check the Inbox on the navigation bar. There are no off-platform notifications for Compliance Forms at this time.
+
+**How do I track the progress of Form completion?**
+
+Great question. We are working on Compliance Forms Analytics that will directly show you the progress of your initiative across the selected Assets. Stay tuned!
+
+### API Tutorials
+
+- [Compliance Form API Guide](../../../api/tutorials/forms.md)
+
+### Related Features
+
+- [DataHub Structured Properties](../../feature-guides/properties/overview.md)
+
+## Next Steps
+
+Now that you have created a DataHub Compliance Form, you're ready to [Complete a Compliance Form](complete-a-form.md).
\ No newline at end of file
diff --git a/docs/features/feature-guides/compliance-forms/overview.md b/docs/features/feature-guides/compliance-forms/overview.md
new file mode 100644
index 0000000000000..86a6d8cc6dadf
--- /dev/null
+++ b/docs/features/feature-guides/compliance-forms/overview.md
@@ -0,0 +1,46 @@
+---
+title: Overview
+---
+
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# About DataHub Compliance Forms
+<FeatureAvailability/>
+
+**DataHub Compliance Forms** streamline the process of documenting, annotating, and classifying your most critical Data Assets through a collaborative, crowdsourced approach.
+
+With Compliance Forms, you can execute large-scale compliance initiatives by assigning tasks (e.g., documentation, tagging, or classification requirements) to the appropriate stakeholders — data owners, stewards, and subject matter experts.
+
+## What are Compliance Forms?
+
+A **Compliance Form** is a flexible and centrally managed tool that enables your data governance or compliance teams to define, enforce, and monitor requirements for specific Data Assets or Columns.
+
+A Compliance Form consists of:
+
+1. **Assets:** The Data Assets or Columns for which the Form must be completed. These represent the scope of the compliance initiative.
+
+2. **Questions:** The set of requirements or conditions that must be completed for each asset. Questions are a vehicle to collect key attributes for your data assets. These can range from simple to complex, with questions that require differing types of answers to complete. Examples include Descriptions, Domains, Owners, Tags, Glossary Terms, and custom Structured Properties.
+
+3. **Assignees:** The users or groups responsible for completing the Form (e.g., asset owners, domain experts, or stewards).
+
+Once a Compliance Form is defined, it can be published. When a form is published, the assignees who are required to complete the requirements will be notified via the Inbox of the tasks that they must complete. In addition, analytics will begin to be gathered about the assets that are meeting or violating the requirements in the form so you can understand your initiative's progress over time.
+
+### Why Use Compliance Forms?
+
+Compliance Forms enable organizations to:  
+- Standardize documentation and metadata across critical Data Assets.  
+- Crowdsource compliance-related tasks to domain experts who are best equipped to provide accurate information.  
+- Scale governance initiatives efficiently while maintaining accuracy and accountability.  
+
+By leveraging Compliance Forms, organizations can ensure consistent metadata quality and foster collaboration between data experts and governance teams.
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/compliance_forms/sample-compliance-form.png" 
+       alt="Sample Compliance Form"/>
+</p>
+
+## Next Steps
+
+Now that you understand the basics of DataHub Compliance Forms, you're ready to [Create a Compliance Form](create-a-form.md).
\ No newline at end of file
diff --git a/docs/features/feature-guides/documentation-forms.md b/docs/features/feature-guides/documentation-forms.md
deleted file mode 100644
index 2edeb8ce302d7..0000000000000
--- a/docs/features/feature-guides/documentation-forms.md
+++ /dev/null
@@ -1,113 +0,0 @@
-import FeatureAvailability from '@site/src/components/FeatureAvailability';
-
-# About DataHub Documentation Forms
-<FeatureAvailability/>
-
-DataHub Documentation Forms streamline the process of setting documentation requirements and delegating annotation responsibilities to the relevant data asset owners, stewards, and subject matter experts.
-
-Forms are highly configurable, making it easy to ask the right questions of the right people, for a specific set of assets.  
-
-## What are Documentation Forms?
-
-You can think of Documentation Forms as a survey for your data assets: a set of questions that must be answered in order for an asset to be considered properly documented.
-
-Verification Forms are an extension of Documentation Forms, requiring a final verification, or sign-off, on all responses before the asset can be considered Verified. This is useful for compliance and/or governance annotation initiatives where you want assignees to provide a final acknowledgement that the information provided is correct.
-
-## Creating and Assigning Documentation Forms
-
-Documentation Forms are defined via YAML with the following details:
-
-- Name and Description to help end-users understand the scope and use case
-- Form Type, either Documentation or Verification
-	- Verification Forms require a final signoff, i.e. Verification, of all required questions before the Form can be considered complete
-- Form Questions (aka "prompts") for end-users to complete
-	- Questions can be assigned at the asset-level and/or the field-level
-	- Asset-level questions can be configured to be required; by default, all questions are optional
-- Assigned Assets, defined by:
-	- A set of specific asset URNs, OR
-	- Assets related to a set of filters, such as Type (Datasets, Dashboards, etc.), Platform (Snowflake, Looker, etc.), Domain (Product, Marketing, etc.), or Container (Schema, Folder, etc.)
-- Optional: Form Assignees
-	- Optionally assign specific DataHub users/groups to complete the Form for all relevant assets
-	- If omitted, any Owner of an Asset can complete Forms assigned to that Asset
-
-Here's an example of defining a Documentation Form via YAML:
-```yaml
-- id: 123456
-  # urn: "urn:li:form:123456"  # optional if id is provided
-  type: VERIFICATION # Supported Types: DOCUMENTATION, VERIFICATION
-  name: "Metadata Initiative 2024"
-  description: "How we want to ensure the most important data assets in our organization have all of the most important and expected pieces of metadata filled out"
-  prompts: # Questions for Form assignees to complete
-    - id: "123"
-      title: "Data Retention Time"
-      description: "Apply Retention Time structured property to form"
-      type: STRUCTURED_PROPERTY
-      structured_property_id: io.acryl.privacy.retentionTime
-      required: True # optional; default value is False
-  entities: # Either pass a list of urns or a group of filters. This example shows a list of urns
-    urns:
-      - urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)
-  # optionally assign the form to a specific set of users and/or groups
-  # when omitted, form will be assigned to Asset owners
-  actors: 
-    users:
-      - urn:li:corpuser:jane@email.com  # note: these should be URNs
-      - urn:li:corpuser:john@email.com
-    groups:
-      - urn:li:corpGroup:team@email.com  # note: these should be URNs
-
-```
-
-:::note
-Documentation Forms currently only support defining Structured Properties as Form Questions
-:::
-
-<!-- ## Completing Documentation Forms -->
-
-<!-- Plain-language instructions of how to use the feature
-
-Provide a step-by-step guide to use feature, including relevant screenshots and/or GIFs
-
-* Where/how do you access it?
-* What best practices exist?
-* What are common code snippets?
- -->
-
-## Additional Resources
-
-### Videos
-
-**Asset Verification in DataHub Cloud**
-
-<p align="center">
-	<iframe width="560" height="315" src="https://www.loom.com/embed/dd834d3cb8f041fca001cea19b2b4071?sid=7073dcd4-407c-41ec-b41d-c99f26dd6a2f" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-</p> 
-
-## FAQ and Troubleshooting
-
-**What is the difference between Documentation and Verification Forms?**
-
-Both form types are a way to configure a set of optional and/or required questions for DataHub users to complete. When using Verification Forms, users will be presented with a final verification step once all required questions have been completed; you can think of this as a final acknowledgement of the accuracy of information submitted.
-
-**Who is able to complete Forms in DataHub?**
-
-By default, any owner of an Asset will be able to respond to questions assigned via a Form.
-
-When assigning a Form to an Asset, you can optionally assign specific DataHub users/groups to fill them out.
-
-**Can I assign multiple Forms to a single asset?**
-
-You sure can! Please keep in mind that an Asset will only be considered Documented or Verified if all required questions are completed on all assiged Forms.
-
-### API Tutorials
-
-- [API Guides on Documentation Form](../../../docs/api/tutorials/forms.md)
-
-:::note
-You must create a Structured Property before including it in a Documentation Form.
-To learn more about creating Structured Properties via CLI, please see the [Create Structured Properties](/docs/api/tutorials/structured-properties.md) tutorial.
-:::
-
-### Related Features
-
-- [DataHub Properties](/docs/features/feature-guides/properties.md)
\ No newline at end of file
diff --git a/docs/features/feature-guides/properties.md b/docs/features/feature-guides/properties.md
deleted file mode 100644
index 0d961b9ceac4f..0000000000000
--- a/docs/features/feature-guides/properties.md
+++ /dev/null
@@ -1,158 +0,0 @@
-import FeatureAvailability from '@site/src/components/FeatureAvailability';
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# About DataHub Properties
-<FeatureAvailability/>
-
-DataHub Custom Properties and Structured Properties are powerful tools to collect meaningful metadata for Assets that might not perfectly fit into other Aspects within DataHub, such as Glossary Terms, Tags, etc. Both types can be found in an Asset's Properties tab:
-
-<p align="center">
-  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/properties/custom_and_structured_properties.png"/>
-</p>
-
-This guide will explain the differences and use cases of each property type.
-
-## What are Custom Properties and Structured Properties?
-Here are the differences between the two property types at a glance:
-
-| Custom Properties | Structured Properties |
-| --- | --- |
-| Map of key-value pairs stored as strings | Validated namespaces and data types |
-| Added to assets during ingestion and via API | Defined via YAML; created and added to assets via CLI |
-| No support for UI-based Edits | Support for UI-based edits |
-
-**Custom Properties** are key-value pairs of strings that capture additional information about assets that is not readily available in standard metadata fields. Custom Properties can be added to assets automatically during ingestion or programmatically via API and *cannot* be edited via the UI.
-<p align="center">
-  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/properties/custom_properties_highlight.png"/>
-</p>
-<p align="center"><em>Example of Custom Properties assigned to a Dataset</em></p>
-
-**Structured Properties** are an extension of Custom Properties, providing a structured and validated way to attach metadata to DataHub Assets. Available as of v0.13.1, Structured Properties have a pre-defined type (Date, Integer, URN, String, etc.). They can be configured to only accept a specific set of allowed values, making it easier to ensure high levels of data quality and consistency. Structured Properties are defined via YAML, added to assets via CLI, and can be edited via the UI.
-<p align="center">
-  <img width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/properties/structured_properties_highlight.png"/>
-</p>
-<p align="center"><em>Example of Structured Properties assigned to a Dataset</em></p>
-
-## Use Cases for Custom Properties and Structured Properties
-**Custom Properties** are useful for capturing raw metadata from source systems during ingestion or programmatically via API. Some examples include:
-
-- GitHub file location of code which generated a dataset
-- Data encoding type
-- Account ID, cluster size, and region where a dataset is stored
-
-**Structured Properties** are useful for setting and enforcing standards of metadata collection, particularly in support of compliance and governance initiatives. Values can be added programmatically via API, then manually via the DataHub UI as necessary. Some examples include:
-
-- Deprecation Date
-  - Type: Date, Single Select
-  - Validation: Must be formatted as 'YYYY-MM-DD'
-- Data Retention Period
-  - Type: String, Single Select
-  - Validation: Adheres to allowed values "30 Days", "90 Days", "365 Days", or "Indefinite"
-- Consulted Compliance Officer, chosen from a list of DataHub users
-  - Type: DataHub User, Multi-Select
-  - Validation: Must be valid DataHub User URN
-
-By using Structured Properties, compliance and governance officers can ensure consistency in data collection across assets.
-
-## Creating, Assigning, and Editing Structured Properties
-
-Structured Properties are defined via YAML, then created and assigned to DataHub Assets via the DataHub CLI. 
-
-Here's how we would define the above examples in YAML:
-
-<Tabs>
-<TabItem value="deprecationDate" label="Deprecation Date" default>
-
-```yaml
-- id: deprecation_date
-  qualified_name: deprecation_date
-  type: date # Supported types: date, string, number, urn, rich_text
-  cardinality: SINGLE # Supported options: SINGLE, MULTIPLE
-  display_name: Deprecation Date
-  description: "Scheduled date when resource will be deprecated in the source system"
-  entity_types: # Define which types of DataHub Assets the Property can be assigned to
-    - dataset
-```
-
-</TabItem>
-<TabItem value="dataRetentionPeriod" label="Data Retention Period">
-
-```yaml
-- id: retention_period
-  qualified_name: retention_period
-  type: string # Supported types: date, string, number, urn, rich_text
-  cardinality: SINGLE # Supported options: SINGLE, MULTIPLE
-  display_name: Data Retention Period
-  description: "Predetermined storage duration before being deleted or archived 
-                based on legal, regulatory, or organizational requirements"
-  entity_types: # Define which types of DataHub Assets the Property can be assigned to
-    - dataset
-  allowed_values:
-    - value: "30 Days"
-      description: "Use this for datasets that are ephemeral and contain PII"
-    - value: "90 Days"
-      description: "Use this for datasets that drive monthly reporting but contain PII"
-    - value: "365 Days"
-      description: "Use this for non-sensitive data that can be retained for longer"
-    - value: "Indefinite"
-      description: "Use this for non-sensitive data that can be retained indefinitely"
-```
-
-</TabItem>
-<TabItem value="consultedComplianceOfficer" label="Consulted Compliance Officer(s)">
-
-```yaml
-- id: compliance_officer
-  qualified_name: compliance_officer
-  type: urn # Supported types: date, string, number, urn, rich_text
-  cardinality: MULTIPLE # Supported options: SINGLE, MULTIPLE
-  display_name: Consulted Compliance Officer(s)
-  description: "Member(s) of the Compliance Team consulted/informed during audit"
-  type_qualifier: # Define the type of Asset URNs to allow
-    - corpuser
-    - corpGroup
-  entity_types: # Define which types of DataHub Assets the Property can be assigned to
-    - dataset
-```
-
-</TabItem>
-</Tabs>
-
-:::note
-To learn more about creating and assigning Structured Properties via CLI, please see the [Create Structured Properties](/docs/api/tutorials/structured-properties.md) tutorial.
-:::
-
-Once a Structured Property is assigned to an Asset, Users with the `Edit Properties` Metadata Privilege will be able to change Structured Property values via the DataHub UI.
-<p align="center">
-  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/properties/edit_structured_properties_modal.png"/>
-</p>
-<p align="center"><em>Example of editing the value of a Structured Property via the UI</em></p>
-
-### Videos
-
-**Deep Dive: UI-Editable Properties**
-
-<p align="center">
-<iframe width="560" height="315" src="https://www.youtube.com/embed/06zaQyKxJYk?si=H_YiwQty25m2xzaP" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
-</p> 
-
-
-### API
-
-Please see the following API guides related to Custom and Structured Properties:
-
-- [Custom Properties API Guide](/docs/api/tutorials/structured-properties.md)
-- [Structured Properties API Guide](/docs/api/tutorials/structured-properties.md)
-
-
-## FAQ and Troubleshooting
-
-**Why can't I edit the value of a Structured Property from the DataHub UI?**
-1. Your version of DataHub does not support UI-based edits of Structured Properties. Confirm you are running DataHub v0.13.1 or later.
-2. You are attempting to edit a Custom Property, not a Structured Property. Confirm you are trying to edit a Structured Property, which will have an "Edit" button visible. Please note that Custom Properties are not eligible for UI-based edits to minimize overwrites during recurring ingestion.
-3. You do not have the necessary privileges. Confirm with your Admin that you have the `Edit Properties` Metadata Privilege.
-
-### Related Features
-
-- [Documentation Forms](/docs/features/feature-guides/documentation-forms.md)
\ No newline at end of file
diff --git a/docs/features/feature-guides/properties/create-a-property.md b/docs/features/feature-guides/properties/create-a-property.md
new file mode 100644
index 0000000000000..2428f70a10551
--- /dev/null
+++ b/docs/features/feature-guides/properties/create-a-property.md
@@ -0,0 +1,261 @@
+---
+title: Create and Add a Structured Property
+---
+
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# Create and Add a DataHub Structured Property
+<FeatureAvailability/>
+
+This guide walks you through creating a Structured Property via the DataHub UI, including:
+
+1. Defining a new Structured Property
+2. Configuring display preferences for a Structured Property
+3. Adding a Structured Property to an Asset
+4. Adding a Structured Property to a Column
+
+:::note
+To learn more about creating and assigning Structured Properties via the CLI, please see the [Create Structured Properties](/docs/api/tutorials/structured-properties.md) tutorial.
+:::
+
+### Prerequisites
+
+To create, edit, or remove Structured Properties, you must have the **View Structured Properties** and **Manage Structured Properties** platform privileges.
+
+To add an existing Structured Property to an Asset, change its value, or remove it from an Asset, you must have the **Edit Properties** metadata privilege.
+
+## Define a New Structured Property
+
+From the navigation bar, go to **Govern** > **Structured Properties**.
+
+Click **+ Create** to start defining your Property.
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/1-list-structured-properties.png"
+       alt="View all Structured Properties"/>
+</p>
+
+First, provide the following details:
+
+1. **Name and Description:** Clearly describe the purpose and meaning of the Structured Property so users understand its role and context.  
+2. **Property Type:** Choose a type that best fits the metadata you want to collect. Available types include **Text**, **Number**, **Date**, **DataHub Entity**, or **Rich Text**. Choosing any of the "List" options allows multiple entries for the Property.
+3. **Allowed Values (Optional):** For **Text**, **Number**, and **DataHub Entity** types, define a set of allowed values to ensure consistent input across assets.
+4. **Applies To:** Specify which DataHub asset types (e.g., Datasets, Dashboards, Pipelines) the Structured Property can be associated with, ensuring relevance and precision.
+
+:::caution
+Once you you save a Structured Property, you **cannot** edit or remove Allowed Values. However, you can add additional Allowed Values.
+:::
+
+For example, imagine your organization wants to standardize how data assets (e.g., Datasets, Tasks, Pipelines) are categorized during their development cycle. By creating a **Lifecycle Stage** Structured Property, you can set a pre-defined list of allowed statuses, such as **Draft**, **Review**, and **Prod**, ensuring consistency and transparency.
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/2.1-create-structured-property-settings.png"
+       alt="View all Structured Properties"/>
+</p>
+
+## Set Display Preferences for the Structured Property
+
+When defining a Structured Property, you can customize how it will be visible to DataHub users. By default, Structured Properties are visible in an Asset's **Properties** tab but can be conditionally configured with the following options:
+
+1. **Hide Property:**  
+   Use this option if the Structured Property contains sensitive metadata that should not be visible to DataHub users via the UI. This ensures that only users with the necessary permissions can view or interact with the property values.
+
+2. **Customize Visibility:**  
+   Decide where the Structured Property appears across the DataHub UI:  
+   - **Asset Badge:** Display the property value as a badge on Assets to highlight key metadata.  
+   - **Asset Sidebar:** Show the property in the Asset Sidebar for quick visibility while navigating an Asset.
+
+3. **Show in Search Filters:**  
+   Enable this option to allow users to filter Assets by the values of this Structured Property. This improves discoverability and facilitates searches for Assets with specific attributes or classifications.  
+
+4. **Show in Columns Table:**  
+   Use this option to display the Structured Property value in the Dataset Schema view’s Columns Table. This is particularly useful for capturing field-level custom metadata and making it accessible alongside schema details.
+
+For the **Lifecycle Stage** example, imagine you want to allow users to filter by lifecycle status and view it at a glance during data discovery. To achieve this, you would enable **Show in Search Filters**, **Asset Badge**, and **Asset Sidebar**:
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/2.2-create-structured-property-display-settings.png"
+       alt="Configure Structured Property Visibility"/>
+</p>
+
+## Add a Structured Property to an Asset
+
+Once a Structured Property has been defined, you can add it to the designated Asset Types.
+
+From an Asset's **Properties** tab, click the `+` button to see a drop-down list of all available Structured Properties. For example, you can now see **Lifecycle Stage** as an option for the `pet_profiles` Dataset:
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/3.1-add-structured-property-entity-level.png"
+       alt="Add a Structured Property to an Asset"/>
+</p>
+
+Continuing with the **Lifecycle Stage** example, designate the `pet_profiles` Dataset as being in **Prod**:
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/3.2-add-structured-property-entity-level-modal.png"
+       alt="Select a Structured Property Value"/>
+</p>
+
+After clicking **Save**, the **Lifecycle Stage** for `pet_profiles` will appear in the following sections of the Asset Page:
+
+1. Properties Tab  
+2. Asset Badge  
+3. Asset Sidebar  
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/3.3-add-structured-property-entity-level-after.png"
+       alt="View a Structured Property on an Asset"/>
+</p>
+
+:::info
+[**DataHub Compliance Forms**](../compliance-forms/overview.md) make it easy to update values for multiple Assets at once!
+:::
+
+### Edit or Remove a Structured Property from an Asset
+
+After a Structured Property has been added to an Asset, users can modify its value or remove the property entirely using the **More** menu:
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/3.4-edit-remove-structured-property-entity-level.png"
+       alt="Editing or Removing a Structured Property on an Asset"/>
+</p>
+
+### Search for Assets by a Structured Property Value
+
+For Structured Properties that have **Show in Search Filters** enabled, users can filter search results based on allowed values.
+
+For example, with the **Lifecycle Stage** property enabled as a filter, users can find it under the **More** dropdown in the Search interface:
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/4.1-search-filter-by-structured-property.png"
+       alt="Structured Property as Search Filter"/>
+</p>
+
+From here, you can quickly narrow down Search results based on the desired stage, such as **Prod**:
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/4.2-search-filter-by-structured-property-results.png"
+       alt="Filtered Results by Structured Property Value"/>
+</p>
+
+Notice how the **Prod** value is displayed prominently on the `pet_profiles` Asset Badge:
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/4.3-search-filter-by-structured-property-results-badge.png"
+       alt="View Structured Property in Asset Badge"/>
+</p>
+
+## Add a Structured Property to a Column
+
+Structured Properties can be applied at the column level, providing deeper context for how individual dataset fields relate to business concepts or terminology. In this example, we’ll create a Structured Property called **Business Label** to help business users understand how dataset columns align with common terminology, acronyms, or key business concepts.
+
+### Define the Business Label Property
+
+Follow these steps to define and configure the **Business Label** Structured Property:
+
+1. **Property Details:**  
+   - **Name:** Business Label  
+   - **Description:** Provide a description to explain its purpose, such as:  
+     *"A user-friendly name for a dataset column, helping business users understand its meaning."*  
+   - **Property Type:** Select **Text**, allowing any valid string to be entered.  
+   - **Applies To:** Set this property to apply exclusively to **Columns**.  
+
+2. **Display Preferences:**  
+   - By default, column-level Structured Properties will be enabled for **all columns** on **all datasets** within DataHub, accessible via the Column Sidebar.  
+   - Optionally, enable **Show in Table Columns** to make the **Business Label** visible within the Columns Table on the dataset schema.  
+
+<p align="center">
+  <img 
+       width="90%"  
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/5.2-create-structured-property-column-display-options.png"
+       alt="Configuring Column Structured Property"/>
+</p>
+
+:::caution
+While the column sidebar provides convenient access to assigned properties, adding too many Structured Properties can clutter the view. Limit the number of properties shown in the sidebar to maintain clarity and usability.
+:::
+
+Once configured, the **Business Label** Structured Property will automatically be added to all columns on dataset assets within DataHub.
+
+For example, after assigning the property, it will appear in two key areas of the `pet_profiles` Asset Page:
+
+1. **Columns Table:** The **Business Label** property and its populated values will be displayed directly within the Columns Table on the Dataset Schema, enabling users to view field-level metadata easily.  
+2. **Column Sidebar:** Structured Properties configured for columns, including **Business Label**, will also appear in the column’s sidebar.  
+
+By applying column-level Structured Properties like **Business Label**, you enhance data discoverability and provide business users with valuable insights while keeping the interface user-friendly.
+
+<p align="center">
+  <img 
+       width="90%"
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/6.1-view-structured-property-column-level.png"
+       alt="Column-level Structured Property in Columns Table"/>
+</p>
+
+### Update the Business Label from the Column Sidebar  
+
+When selecting a specific column in the UI, the **Business Label** Structured Property will be visible in the column’s sidebar. Users with appropriate permissions can view or update the value directly from this interface.
+
+<p align="center">
+  <img 
+       width="90%"
+       src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/6.2-view-structured-property-column-level-sidebar.png"
+       alt="Column-level Structured Property in Sidebar"/>
+</p>
+
+This setup ensures that column-specific metadata, such as the **Business Label**, is accessible and actionable, helping business users better understand the dataset's structure and its alignment with key business concepts.
+
+## FAQ and Troubleshooting
+
+### Why can’t I change a Structured Property’s definition?
+
+Once a Structured Property has been defined, only certain aspects can be modified:
+
+**You can change:**
+- Title and description
+- Add new allowed values
+- Add new supported asset types
+- Update display preferences
+
+**You cannot change:**
+- The type of the Structured Property
+- Existing allowed values and their definitions
+
+### Why can't I configure a Structured Property to appear as an Asset Badge?
+- Only **Text** and **Number** types with allowed values can be configured as Asset Badges.
+- Only one Structured Property can be displayed as a Badge for a given Asset.
+
+### Why can't I filter Search Results by a Structured Property?
+- Verify that the Structured Property has been configured to appear in search filters.
+- Ensure the filter is relevant by checking if there are assets associated with the Structured Property's value in your search results. Try different search terms or relax other applied filters.
+
+### Why can't I add a Structured Property to an Asset?
+- Confirm you have the **Edit Properties** privilege.
+- Ensure the Structured Property has already been created and supports the type of Asset you're trying to modify.
+
+### API Tutorials
+
+- [Structured Properties API Guide](/docs/api/tutorials/structured-properties.md)
+
+### Related Features
+
+- [DataHub Compliance Forms](/docs/features/feature-guides/compliance-forms/overview.md)
\ No newline at end of file
diff --git a/docs/features/feature-guides/properties/overview.md b/docs/features/feature-guides/properties/overview.md
new file mode 100644
index 0000000000000..7637a3be53e0d
--- /dev/null
+++ b/docs/features/feature-guides/properties/overview.md
@@ -0,0 +1,54 @@
+---
+title: Overview
+---
+
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# About DataHub Structured Properties
+<FeatureAvailability/>
+
+DataHub **Structured Properties** allow you to add custom, validated properties to any Entity type in DataHub. Using Structured Properties, you can enable data discovery and governance based on attributes unique to your organization.
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/properties/custom_and_structured_properties.png"/>
+</p>
+
+## What are Structured Properties?
+
+**Structured Properties** are a powerful way to customize your DataHub environment, enabling you to align metadata with your organization’s unique needs. By defining specific property types—such as Date, Integer, DataHub Asset, or Text—you can apply meaningful, context-aware attributes to your Assets. Validation rules, like restricting allowed values or enforcing specific formats, ensure consistency while giving you the flexibility to reflect your business’s terminology, workflows, and priorities.
+
+Structured Properties can be added to the following Asset Types:
+
+- Data Assets, such as Datasets, Columns, Tasks, Pipelines, Charts, Dashboards, and more.
+- DataHub Entities, such as Domains, Glossary Terms & Groups, and Data Products.
+
+### Key Features of Structured Properties:
+
+1. **Typed Fields:** Properties are explicitly typed, including options like Date, Integer, URN, or Text.
+2. **Allowed Values:** Enforce standards by restricting values to a specific format or a pre-defined list of acceptable inputs.
+3. **Targeted Application:** Structured Properties can be tailored to specific asset types—such as Datasets, Columns, or Dashboards—ensuring they align with your organization’s data management needs and usage context.
+
+### Display Settings
+
+Structured Properties offer several configuration options to enhance metadata management:
+
+- **Hide Property:** For use cases where property values should not be viewable by DataHub users.
+- **Show in Search Filters:** Enables users to filter for Assets based on specific property values, improving discoverability.
+- **Customize Visibility:** Allows you to control where the Structured Property appears, such as in the Asset Badge, Asset Sidebar, and/or a Dataset Schema view’s Columns Table.
+
+## Why Use Structured Properties?
+
+Structured Properties are especially useful for organizations that require:
+
+- **Customization:** Customize how your end-users find assets within DataHub.
+- **Governance and Compliance:** Collect metadata in a way that supports compliance with internal or external standards.
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/structured_properties/1-list-structured-properties.png"/>
+</p>
+
+By leveraging these configurations, teams can ensure their metadata adheres to organizational policies and improves the discoverability and usability of Data Assets.
+
+## Next Steps
+
+Now that you understand Structured Properties, you’re ready to [Create a Structured Property](create-a-property.md).
\ No newline at end of file
diff --git a/docs/how/delete-metadata.md b/docs/how/delete-metadata.md
index f720a66ce5765..e36940bf39835 100644
--- a/docs/how/delete-metadata.md
+++ b/docs/how/delete-metadata.md
@@ -4,7 +4,7 @@
 To follow this guide, you'll need the [DataHub CLI](../cli.md).
 :::
 
-There are a two ways to delete metadata from DataHub:
+There are two ways to delete metadata from DataHub:
 
 1. Delete metadata attached to entities by providing a specific urn or filters that identify a set of urns (delete CLI).
 2. Delete metadata created by a single ingestion run (rollback).
@@ -233,7 +233,13 @@ To view the ids of the most recent set of ingestion batches, execute
 datahub ingest list-runs
 ```
 
-That will print out a table of all the runs. Once you have an idea of which run you want to roll back, run
+That will print out a table of all the runs. To see run statuses or to filter runs by URN/source run
+
+```shell
+datahub ingest list-source-runs
+```
+
+Once you have an idea of which run you want to roll back, run
 
 ```shell
 datahub ingest show --run-id <run-id>
diff --git a/docs/how/search.md b/docs/how/search.md
index 2274fe7c09240..652338f8f581a 100644
--- a/docs/how/search.md
+++ b/docs/how/search.md
@@ -359,6 +359,33 @@ queryConfigurations:
       boost_mode: multiply
 ```
 
+Similar example to boost with `primary` AND `gold` instead of the previous OR condition.
+
+```yaml
+queryConfigurations:
+  - queryRegex: .*
+    
+    simpleQuery: true
+    prefixMatchQuery: true
+    exactMatchQuery: true
+
+    functionScore:
+      functions:
+
+        - filter:
+            bool:
+              filter:
+                - term:
+                    tags.keyword: urn:li:tag:primary
+                - term:
+                    tags.keyword: urn:li:tag:gold
+          weight: 3.0
+
+
+      score_mode: multiply
+      boost_mode: multiply
+```
+
 ##### Example 2: Preferred Data Platform
 
 Boost the `urn:li:dataPlatform:hive` platform.
diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 00e020bd2a387..5bc0e66fa2ff1 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -18,12 +18,52 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ## Next
 
+- #11560 - The PowerBI ingestion source configuration option include_workspace_name_in_dataset_urn determines whether the workspace name is included in the PowerBI dataset's URN.<br/> PowerBI allows to have identical name of semantic model and their tables across the workspace, It will overwrite the semantic model in-case of multi-workspace ingestion.<br/>
+  Entity urn with `include_workspace_name_in_dataset_urn: false`
+
+  ```
+   urn:li:dataset:(urn:li:dataPlatform:powerbi,[<PlatformInstance>.]<SemanticModelName>.<TableName>,<ENV>)
+  ```
+
+  Entity urn with `include_workspace_name_in_dataset_urn: true`
+
+  ```
+   urn:li:dataset:(urn:li:dataPlatform:powerbi,[<PlatformInstance>.].<WorkspaceName>.<SemanticModelName>.<TableName>,<ENV>)
+  ```
+
+  The config `include_workspace_name_in_dataset_urn` is default to `false` for backward compatibility, However, we recommend enabling this flag after performing the necessary cleanup.
+  If stateful ingestion is enabled, running ingestion with the latest CLI version will handle the cleanup automatically. Otherwise, we recommend soft deleting all powerbi data via the DataHub CLI:
+  `datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true.
+
+- #11701: The Fivetran `sources_to_database` field is deprecated in favor of setting directly within `sources_to_platform_instance.<key>.database`.
+- #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information.
+- #12056: The DataHub Airflow plugin no longer supports Airflow 2.1 and Airflow 2.2.
+- #12056: The DataHub Airflow plugin now defaults to the v2 plugin implementation.
+- #12077: `Kafka` source no longer ingests schemas from schema registry as separate entities by default, set `ingest_schemas_as_entities` to `true` to ingest them
+- OpenAPI Update: PIT Keep Alive parameter added to scroll. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set.
+- OpenAPI Update 2: Previously there was an incorrectly marked parameter named `sort` on the generic list entities endpoint for v3. This parameter is deprecated and only supports a single string value while the documentation indicates it supports a list of strings. This documentation error has been fixed and the correct field, `sortCriteria`, is now documented which supports a list of strings. 
+
 ### Breaking Changes
 
-- #11486 - Deprecated Criterion filters using `value`. Use `values` instead. This also deprecates the ability to use comma delimited string to represent multiple values using `value`.
+- #11486 - Criterion's `value` parameter has been previously deprecated. Use of `value` instead of `values` is no longer supported and will be completely removed on the next major version.
 - #11484 - Metadata service authentication enabled by default
 - #11484 - Rest API authorization enabled by default
 - #10472 - `SANDBOX` added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases.
+- #11619 - schema field/column paths can no longer be empty strings
+- #11619 - schema field/column paths can no longer be duplicated within the schema
+- #11570 - The `DatahubClientConfig`'s server field no longer defaults to `http://localhost:8080`. Be sure to explicitly set this.
+- #11570 - If a `datahub_api` is explicitly passed to a stateful ingestion config provider, it will be used. We previously ignored it if the pipeline context also had a graph object.
+- #11518 - DataHub Garbage Collection: Various entities that are soft-deleted (after 10d) or are timeseries _entities_ (dataprocess, execution requests) will be removed automatically using logic in the `datahub-gc` ingestion source.
+- #12020 - Removed `sql_parser` configuration from the Redash source, as Redash now exclusively uses the sqlglot-based parser for lineage extraction.
+- #12020 - Removed `datahub.utilities.sql_parser`, `datahub.utilities.sql_parser_base` and `datahub.utilities.sql_lineage_parser_impl` module along with `SqlLineageSQLParser` and `DefaultSQLParser`. Use `create_lineage_sql_parsed_result` from `datahub.sql_parsing.sqlglot_lineage` module instead.
+- #11518 - DataHub Garbage Collection: Various entities that are soft-deleted
+  (after 10d) or are timeseries *entities* (dataprocess, execution requests)
+  will be removed automatically using logic in the `datahub-gc` ingestion
+  source.
+- #12067 - Default behavior of DataJobPatchBuilder in Python sdk has been
+  changed to NOT fill out `created` and `lastModified` auditstamps by default
+  for input and output dataset edges. This should not have any user-observable
+  impact (time-based lineage viz will still continue working based on observed time), but could break assumptions previously being made by clients.
 
 ### Potential Downtime
 
@@ -56,6 +96,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 - #11313 - `datahub get` will no longer return a key aspect for entities that don't exist.
 - #11369 - The default datahub-rest sink mode has been changed to `ASYNC_BATCH`. This requires a server with version 0.14.0+.
 - #11214 Container properties aspect will produce an additional field that will require a corresponding upgrade of server. Otherwise server can reject the aspects.
+- #10190 - `extractor_config.set_system_metadata` of `datahub` source has been moved to be a top level config in the recipe under `flags.set_system_metadata`
 
 ### Potential Downtime
 
@@ -64,6 +105,9 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 ### Other Notable Changes
 
 - Downgrade to previous version is not automatically supported.
+- Data Product Properties Unset side effect introduced
+  - Previously, Data Products could be set as linked to multiple Datasets if modified directly via the REST API rather than linked through the UI or GraphQL. This side effect aligns the REST API behavior with the GraphQL behavior by introducting a side effect that enforces the 1-to-1 constraint between Data Products and Datasets
+  - NOTE: There is a pathological pattern of writes for Data Products that can introduce issues with write processing that can occur with this side effect. If you are constantly changing all of the Datasets associated with a Data Product back and forth between multiple Data Products it will result in a high volume of writes due to the need to unset previous associations.
 
 ## 0.14.0.2
 
diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md
index aca6d30619ea8..345213a0672d3 100644
--- a/docs/lineage/airflow.md
+++ b/docs/lineage/airflow.md
@@ -13,14 +13,14 @@ The DataHub Airflow plugin supports:
 - Task run information, including task successes and failures.
 - Manual lineage annotations using `inlets` and `outlets` on Airflow operators.
 
-There's two actively supported implementations of the plugin, with different Airflow version support.
+There's two implementations of the plugin, with different Airflow version support.
 
-| Approach  | Airflow Version | Notes                                                                       |
-| --------- | --------------- | --------------------------------------------------------------------------- |
-| Plugin v2 | 2.3.4+          | Recommended. Requires Python 3.8+                                           |
-| Plugin v1 | 2.1 - 2.8       | No automatic lineage extraction; may not extract lineage if the task fails. |
+| Approach  | Airflow Versions | Notes                                                                                   |
+| --------- | ---------------- | --------------------------------------------------------------------------------------- |
+| Plugin v2 | 2.3.4+           | Recommended. Requires Python 3.8+                                                       |
+| Plugin v1 | 2.3 - 2.8        | Deprecated. No automatic lineage extraction; may not extract lineage if the task fails. |
 
-If you're using Airflow older than 2.1, it's possible to use the v1 plugin with older versions of `acryl-datahub-airflow-plugin`. See the [compatibility section](#compatibility) for more details.
+If you're using Airflow older than 2.3, it's possible to use the v1 plugin with older versions of `acryl-datahub-airflow-plugin`. See the [compatibility section](#compatibility) for more details.
 
 <!-- TODO: Update the local Airflow guide and link to it here. -->
 <!-- If you are looking to run Airflow and DataHub using docker locally, follow the guide [here](../../docker/airflow/local_airflow.md). -->
@@ -29,7 +29,7 @@ If you're using Airflow older than 2.1, it's possible to use the v1 plugin with
 
 ### Installation
 
-The v2 plugin requires Airflow 2.3+ and Python 3.8+. If you don't meet these requirements, use the v1 plugin instead.
+The v2 plugin requires Airflow 2.3+ and Python 3.8+. If you don't meet these requirements, see the [compatibility section](#compatibility) for other options.
 
 ```shell
 pip install 'acryl-datahub-airflow-plugin[plugin-v2]'
@@ -84,9 +84,10 @@ enabled = True  # default
 
 ### Installation
 
-The v1 plugin requires Airflow 2.1 - 2.8 and Python 3.8+. If you're on older versions, it's still possible to use an older version of the plugin. See the [compatibility section](#compatibility) for more details.
+The v1 plugin requires Airflow 2.3 - 2.8 and Python 3.8+. If you're on older versions, it's still possible to use an older version of the plugin. See the [compatibility section](#compatibility) for more details.
 
-If you're using Airflow 2.3+, we recommend using the v2 plugin instead. If you need to use the v1 plugin with Airflow 2.3+, you must also set the environment variable `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN=true`.
+Note that the v1 plugin is less featureful than the v2 plugin, and is overall not actively maintained.
+Since datahub v0.15.0, the v2 plugin has been the default. If you need to use the v1 plugin with `acryl-datahub-airflow-plugin` v0.15.0+, you must also set the environment variable `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN=true`.
 
 ```shell
 pip install 'acryl-datahub-airflow-plugin[plugin-v1]'
@@ -145,6 +146,7 @@ conn_id = datahub_rest_default  # or datahub_kafka_default
 | datajob_url_link           | taskinstance         | If taskinstance, the datajob url will be taskinstance link on airflow. It can also be grid.                                                                                            |
 |                            |
 | graceful_exceptions        | true                 | If set to true, most runtime errors in the lineage backend will be suppressed and will not cause the overall task to fail. Note that configuration issues will still throw exceptions. |
+| dag_filter_str             | { "allow": [".*"] }  | AllowDenyPattern value in form of JSON string to filter the DAGs from running.                                                                                                         |
 
 #### Validate that the plugin is working
 
@@ -162,7 +164,7 @@ Only the v2 plugin supports automatic lineage extraction. If you're using the v1
 To automatically extract lineage information, the v2 plugin builds on top of Airflow's built-in [OpenLineage extractors](https://openlineage.io/docs/integrations/airflow/default-extractors).
 As such, we support a superset of the default operators that Airflow/OpenLineage supports.
 
-The SQL-related extractors have been updated to use [DataHub's SQL lineage parser](https://blog.datahubproject.io/extracting-column-level-lineage-from-sql-779b8ce17567), which is more robust than the built-in one and uses DataHub's metadata information to generate column-level lineage.
+The SQL-related extractors have been updated to use [DataHub's SQL lineage parser](./sql_parsing.md), which is more robust than the built-in one and uses DataHub's metadata information to generate column-level lineage.
 
 Supported operators:
 
@@ -190,6 +192,10 @@ These operators are supported by OpenLineage, but we haven't tested them yet:
 There's also a few operators (e.g. BashOperator, PythonOperator) that have custom extractors, but those extractors don't generate lineage.
 -->
 
+Known limitations:
+
+- We do not fully support operators that run multiple SQL statements at once. In these cases, we'll only capture lineage from the first SQL statement.
+
 ## Manual Lineage Annotation
 
 ### Using `inlets` and `outlets`
@@ -333,13 +339,45 @@ TypeError: on_task_instance_success() missing 3 required positional arguments: '
 
 The solution is to upgrade `acryl-datahub-airflow-plugin>=0.12.0.4` or upgrade `pluggy>=1.2.0`. See this [PR](https://github.com/datahub-project/datahub/pull/9365) for details.
 
+### Disabling the DataHub Plugin v2
+
+There are two ways to disable the DataHub Plugin v2:
+
+#### 1. Disable via Configuration
+
+Set the `datahub.enabled` configuration property to `False` in the `airflow.cfg` file and restart the Airflow environment to reload the configuration and disable the plugin.
+
+```ini title="airflow.cfg"
+[datahub]
+enabled = False
+```
+
+#### 2. Disable via Airflow Variable (Kill-Switch)
+
+If a restart is not possible and you need a faster way to disable the plugin, you can use the kill-switch. Create and set the `datahub_airflow_plugin_disable_listener` Airflow variable to `true`. This ensures that the listener won't process anything.
+
+#### Command Line
+
+```shell
+airflow variables set datahub_airflow_plugin_disable_listener true
+```
+
+#### Airflow UI
+
+1. Go to Admin -> Variables.
+2. Click the "+" symbol to create a new variable.
+3. Set the key to `datahub_airflow_plugin_disable_listener` and the value to `true`.
+
+This will immediately disable the plugin without requiring a restart.
+
 ## Compatibility
 
-We no longer officially support Airflow <2.1. However, you can use older versions of `acryl-datahub-airflow-plugin` with older versions of Airflow.
-Both of these options support Python 3.7+.
+We no longer officially support Airflow <2.3. However, you can use older versions of `acryl-datahub-airflow-plugin` with older versions of Airflow.
+The first two options support Python 3.7+, and the last option supports Python 3.8+.
 
 - Airflow 1.10.x, use DataHub plugin v1 with acryl-datahub-airflow-plugin <= 0.9.1.0.
 - Airflow 2.0.x, use DataHub plugin v1 with acryl-datahub-airflow-plugin <= 0.11.0.1.
+- Airflow 2.2.x, use DataHub plugin v2 with acryl-datahub-airflow-plugin <= 0.14.1.5.
 
 DataHub also previously supported an Airflow [lineage backend](https://airflow.apache.org/docs/apache-airflow/2.2.0/lineage.html#lineage-backend) implementation. While the implementation is still in our codebase, it is deprecated and will be removed in a future release.
 Note that the lineage backend did not support automatic lineage extraction, did not capture task failures, and did not work in AWS MWAA.
diff --git a/docs/lineage/sql_parsing.md b/docs/lineage/sql_parsing.md
new file mode 100644
index 0000000000000..76161de0040f4
--- /dev/null
+++ b/docs/lineage/sql_parsing.md
@@ -0,0 +1,63 @@
+---
+title: SQL Parsing
+---
+
+# The DataHub SQL Parser
+
+Many data platforms are built on top of SQL, which means deeply understanding SQL queries is critical for understanding column-level lineage, usage, and more.
+
+DataHub's SQL parser is built on top of [sqlglot](https://github.com/tobymao/sqlglot) and adds a number of additional features to improve the accuracy of SQL parsing.
+
+In our benchmarks, the DataHub SQL parser generates lineage with 97-99% accuracy and outperforms other SQL parsers by a wide margin.
+
+We've published a blog post on some of the technical details of the parser: [Extracting Column Lineage from SQL Queries](https://blog.datahubproject.io/extracting-column-level-lineage-from-sql-779b8ce17567).
+
+## Built-in SQL Parsing Support
+
+If you're using a tool that DataHub already [integrates with](https://datahubproject.io/integrations), check the documentation for that specific integration.
+Most of our integrations, including Snowflake, BigQuery, Redshift, dbt, Looker, PowerBI, Airflow, etc, use the SQL parser to generate column-level lineage and usage statistics.
+
+If you’re using a different database system for which we don’t support column-level lineage out of the box, but you do have a database query log available, the [SQL queries](../generated/ingestion/sources/sql-queries.md) connector can generate column-level lineage and table/column usage statistics from the query log.
+
+## SDK Support
+
+Our SDK provides a [`DataHubGraph.parse_sql_lineage()`](../../python-sdk/clients.md#datahub.ingestion.graph.client.DataHubGraph.parse_sql_lineage) method for programmatically parsing SQL queries.
+
+The resulting object contains a `sql_parsing_result.debug_info.confidence_score` field, which is a 0-1 value indicating the confidence of the parser.
+
+There are also a number of utilities in the `datahub.sql_parsing` module. The `SqlParsingAggregator` is particularly useful, as it can also resolve lineage across temp tables and table renames/swaps.
+Note that these utilities are not officially part of the DataHub SDK and hence do not have the same level of stability and support as the rest of the SDK.
+
+## Capabilities
+
+### Supported
+
+- Table-level lineage for `SELECT`, `CREATE`, `INSERT`, `UPDATE`, `DELETE`, and `MERGE` statements
+- Column-level lineage for `SELECT` (including `SELECT INTO`), `CREATE VIEW`, `CREATE TABLE AS SELECT` (CTAS), `INSERT`, and `UPDATE` statements
+- Subqueries
+- CTEs
+- `UNION ALL` constructs - will merge lineage across the clauses of the `UNION`
+- `SELECT *` and similar expressions will automatically be expanded with the table schemas registered in DataHub. This includes support for platform instances.
+- Automatic handling for systems where table and column names are case insensitive. Generally requires that `convert_urns_to_lowercase` is enabled when the corresponding table schemas were ingested into DataHub.
+  - Specifically, we'll do fuzzy matching against the table names and schemas to resolve the correct URNs. We do not support having multiple tables/columns that only differ in casing.
+- For BigQuery, sharded table suffixes will automatically be normalized. For example, `proj.dataset.table_20230616` will be normalized to `proj.dataset.table_yyyymmdd`. This matches the behavior of our BigQuery ingestion connector, and hence will result in lineage linking up correctly.
+
+### Not supported
+
+- Scalar `UDFs` - We will generate lineage pointing at the columns that are inputs to the UDF, but will not be able to understand the UDF itself.
+- Tabular `UDFs`
+- `json_extract` and similar functions
+- `UNNEST` - We will do a best-effort job, but cannot reliably generate column-level lineage in the presence of `UNNEST` constructs.
+- Structs - We will do a best-effort attempt to resolve struct subfields, but it is not guaranteed. This will only impact column-level lineage.
+- Snowflake's multi-table inserts
+- Multi-statement SQL / SQL scripting
+
+### Limitations
+
+- We only support the 20+ SQL dialects supported by the underlying [sqlglot](https://github.com/tobymao/sqlglot) library.
+- There's a few SQL syntaxes that we don't support yet, but intend to support in the future.
+  - `INSERT INTO (col1_new, col2_new) SELECT col1_old, col2_old FROM ...`. We only support `INSERT INTO` statements that either (1) don't specify a column list, or (2) specify a column list that matches the columns in the `SELECT` clause.
+  - `MERGE INTO` statements - We don't generate column-level lineage for these.
+- In cases where the table schema information in DataHub is outdated or otherwise incorrect, we may not be able to generate accurate column-level lineage.
+- We trip over BigQuery queries that use the `_partitiontime` and `_partitiondate` pseudo-columns with a table name prefix e.g. `my_table._partitiontime` fails. However, unqualified references like `_partitiontime` and `_partitiondate` will be fine.
+- We do not consider columns referenced in `WHERE`, `GROUP BY`, `ORDER BY`, etc. clauses to be part of lineage. For example, `SELECT col1, col2 FROM upstream_table WHERE col3 = 3` will not generate any lineage related to `col3`.
diff --git a/docs/managed-datahub/integrations/oidc-sso-integration.md b/docs/managed-datahub/integrations/oidc-sso-integration.md
index ec4ca311a0de5..834eb55f6daf5 100644
--- a/docs/managed-datahub/integrations/oidc-sso-integration.md
+++ b/docs/managed-datahub/integrations/oidc-sso-integration.md
@@ -6,44 +6,42 @@ description: >-
 import FeatureAvailability from '@site/src/components/FeatureAvailability';
 
 
-# OIDC SSO Integration
+# Enable OIDC SSO
 <FeatureAvailability saasOnly />
 
+This guide will walk you through configuring OIDC Single Sign-On in DataHub Cloud.
 
-_Note that we do not yet support LDAP or SAML authentication. Please let us know if either of these integrations would be useful for your organization._
+### Step 1. Complete OIDC Prerequisites
 
-If you'd like to do a deeper dive into OIDC configuration outside of the UI, please see our docs [here](/docs/authentication/guides/sso/configure-oidc-react.md)
+You will need the following in order to enable OIDC SSO in DataHub Cloud:
 
-### Getting Details From Your Identity Provider
+- Gather the **Client ID**, **Client Secret**, and **Discovery URI** for your OIDC provider, as detailed in [this guide](../../authentication/guides/sso/initialize-oidc.md).
+- Confirm you have the `Manage Platform Settings` privilege in DataHub.
 
-To set up the OIDC integration, you will need the following pieces of information.
 
-1. _Client ID_ - A unique identifier for your application with the identity provider
-2. _Client Secret_ - A shared secret to use for exchange between you and your identity provider.
-3. _Discovery URL_ - A URL where the OIDC API of your identity provider can be discovered. This should suffixed by `.well-known/openid-configuration`. Sometimes, identity providers will not explicitly include this URL in their setup guides, though this endpoint will exist as per the OIDC specification. For more info see [here](http://openid.net/specs/openid-connect-discovery-1_0.html).
+### Step 2. Enable OIDC SSO
 
-The callback URL to register in your Identity Provider will be
+1. In DataHub Cloud, navigate to **Settings > Platform > SSO** and choose **OIDC**.
 
-```
-https://<your-acryl-domain>.acryl.io/callback/oidc 
-```
-
-### Configuring OIDC SSO
-
-> In order to set up the OIDC SSO integration, the user must have the `Manage Platform Settings` privilege.
-
-#### Enabling the OIDC Integration
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/saas/configure-oidc.png"/>
+</p>
 
-To enable the OIDC integration, start by navigating to **Settings > Platform > SSO.**
+2. Enter the **Client ID**, **Client Secret**, and **Discovery URI** from Step 1.
+3. Confirm your preferred **User Provisioning Strategy**:
 
-1. Click **OIDC**
-2. Enable the Integration
-3. Enter the **Client ID, Client Secret, and Discovery URI** obtained in the previous steps
-4. If there are any advanced settings you would like to configure, click on the **Advanced** button. These come with defaults, so only input settings here if there is something you need changed from the default configuration.
-5. Click **Update** to save your settings.
+    * **Just-in-Time (JIT) Provisioning** is enabled by default, automatically creating a DataHub User on login if one does not exist.
+    * **Pre-Provisioning DataHub Users** will only allow login for pre-provisioned DataHub Users. _Requires configuring SSO Ingestion._
 
+4. Optionally enable **Extract Groups** to extract group memberships in the OIDC profile by default. _Requires JIT Provisioning._
+5. Click **Connect**.
+6. Log out and log back in through SSO to confirm connection succeeded.
 
 <p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/image-(10).png"/>
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/image-(10).png"/>
 </p>
 
+
+:::note
+We do not yet support LDAP or SAML authentication. Please let us know if either of these integrations would be useful for your organization.
+:::
\ No newline at end of file
diff --git a/docs/managed-datahub/release-notes/v_0_3_6.md b/docs/managed-datahub/release-notes/v_0_3_6.md
index cf1c9ff4e8c6a..e513a46ace4c5 100644
--- a/docs/managed-datahub/release-notes/v_0_3_6.md
+++ b/docs/managed-datahub/release-notes/v_0_3_6.md
@@ -48,6 +48,15 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies
     - Impact analysis and data product assets pages properly show results based on the currently selected view
     - The New Users section in the Analytics page now tracks actual new users instead of last login
     - Data Contracts builder now shows custom assertions as expected
+    - "View in Platform" buttons link location fixed 
+    - Compact search cards unnecessary title truncation fixed 
+    - Domain contents summary updates after domain is changed, without refresh 
+    - Incidents tab updates after an incident is created, updated, or resolved, without refresh 
+    - On lineage visualization, fix lineage expansion through transformation nodes while hiding transformations 
+    - On lineage visualization, fix positioning of query nodes on lineage expansion 
+    - On lineage visualization, cycles in graphs should no longer result in disappearing edges 
+    - Remote executor cleans up temporary files to avoid disk memory leak
+    - Fixed hover cards when using Compact Card View
 
 - Product changes
     - Show total (nested) asset count on home page Domain cards
@@ -68,6 +77,11 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies
     - Add "Last Updated (in Source)" filter to search
     - Add "View In \[Source]" button to lineage visualization sidebar
     - Send test notification button will always be visible in the subscriptions side bar if the user has set their memberID
+    - On sibling combined entity page, viewing different schema versions no longer supported
+    - On sibling combined entity page, change history drawer reflects changes to both siblings
+    - On sibling combined entity sidebar, lineage summary is hidden to avoid confusion
+    - For siblings combined entity page or search / hover card, multiple "View in Platform" icons should appear if applicable 
+    - Search cards configured to better handle smaller widths
 
 - Ingestion changes
 
diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md
new file mode 100644
index 0000000000000..31302403ea930
--- /dev/null
+++ b/docs/managed-datahub/release-notes/v_0_3_7.md
@@ -0,0 +1,191 @@
+# v0.3.7
+---
+
+Release Availability Date
+---
+13-Nov-2024
+
+Recommended CLI/SDK
+---
+- `v0.14.1.12` with release notes at https://github.com/datahub/datahub/releases/tag/v0.14.1.12
+
+If you are using an older CLI/SDK version, then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, GitHub Actions, Airflow, in Python SDK somewhere, Java SDK, etc. This is a strong recommendation to upgrade, as we keep on pushing fixes in the CLI, and it helps us support you better.
+
+## Known Issues
+
+### v0.3.7.8
+   * Notes Feature
+      * Adding a Note to an entity will result in that note showing up in the Settings > Home Page list of announcements as well as the profile page of the entity.
+      * If more than 30 Notes are added to entities, there's a risk that home page announcements will not show up on the home page properly.
+      * Notes are only supported for Dataset and Column entities in this release.
+
+### v0.3.7.7
+   * Postgres regression, non-functional when using postgres
+
+### v0.3.7.3
+   * Search page fails to render when filters are applied with a query which returns zero results.
+
+## Release Changelog
+---
+
+### v0.3.7.8
+
+- Helm Chart Requirement: 1.4.157+
+- [Postgres] Fix regression from MySQL fix in v0.3.7.7
+- [UI] Fix editing post on entity profile page becomes announcement
+
+### v0.3.7.7
+
+- [UI] Fix bug showing upstream lineage dbt source leaves
+- [UI] Show column-level lineage through transformational home node
+- [UI] Browse nodes titles expand to full width of panel
+- [UI] Data product preview cards display correctly
+- [UI] Fix elasticsearch usage sort field names
+- [UI] Add structured property display settings feature
+- [Executor] Fix false errors on cli ingestions
+- [Search] Schema field boost reduced
+- [Search] Search usage ranking null_fill fix
+- [Search] Single term with underscores by default no longer considered quoted
+- [Metadata Tests] Metadata Test shutdown actions flush
+- [Metadata Tests] Add deduplicate logic for MCP batches
+- [Metadata Tests] Prevent mutation of systemMetadata in patch batches
+- [MAE Consumer] Fix graph edge on container delete exception
+- [Notifications] Filter out system ingestion source notifications
+- [MySQL] Fix index gap lock deadlock
+- [API] DataJobInputOutput finegrained lineage fix
+
+### v0.3.7.6
+
+- [UI] fix(automations): white screen automations with dbt sync
+
+### v0.3.7.5
+
+- [GMS] Fix upstream lineage patching when path contained encoded slash
+- [UI] Fix merging siblings schema with v1 and v2 fields
+- [UI] Fix display nullable in schema field drawer
+- [Ingestion] Reduce Data Product write volume from unset side-effect
+
+### v0.3.7.4
+
+- [#11935](https://github.com/datahub-project/datahub/pull/11935) - Added environment variable for enabling stricter URN validation rules `STRICT_URN_VALIDATION_ENABLED` [[1](https://datahubproject.io/docs/what/urn/#restrictions)].
+- [Automations] Filter out self-nodes in glossary term propagation
+- [Remote Executor] Allow dashes in executor ids.
+- [Search] Fix Nested Filter Counts in Primary Search
+- [Search] Fix white screen of death on empty search result
+- [Columns Tab] Support searching nested struct columns correctly in V2 UI.
+- [Logo] Fix fit of custom logo for V2 UI nav bar.
+- [Structured Properties] Better handling for special characters in structured properties
+- [Lineage] Improvements to handling lineage cycles
+- [Metadata Tests] Improve Reliability of Metadata Tests Action Application
+- [Slack Integration] Minor improvement in authentication redirect to integrate with Slack
+- [Columns Tab] Property display nullable status in column sidebar (bug)
+- [Columns Tab] Fixing merging of sibling schemas between V2 and V1 field paths.
+- [Documentation] Support group authors for institutional memory aspect
+
+
+### v0.3.7
+
+- All changes in https://github.com/datahub-project/datahub/releases/tag/v0.14.1
+    - Note Breaking Changes: https://datahubproject.io/docs/how/updating-datahub/#0141
+
+- Breaking Changes
+    - Authentication & RestAPI Authorization enabled by default (since v0.3.6)
+    - Helm Chart Requirement: 1.4.137+
+        - Recommend setting timezone for `datahub-gc` and `datahub-usage-reporting`  
+        - ```yaml
+          acryl-datahub:
+            global:
+              datahub:
+                timezone: 'America/Los_Angeles'
+          ```
+    - #11486 - Criterion's `value` parameter has been previously deprecated. Use of `value` instead of `values` is no longer supported and will be completely removed on the next major version.
+    - #10472 - `SANDBOX` added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases.
+    - #11619 - schema field/column paths can no longer be empty strings
+    - #11619 - schema field/column paths can no longer be duplicated within the schema
+    - #11570 - The `DatahubClientConfig`'s server field no longer defaults to `http://localhost:8080`. Be sure to explicitly set this.
+    - #11570 - If a `datahub_api` is explicitly passed to a stateful ingestion config provider, it will be used. We previously ignored it if the pipeline context also had a graph object.
+    - #11518 - DataHub Garbage Collection: Various entities that are soft-deleted (after 10d) or are timeseries *entities* (dataprocess, execution requests) will be removed automatically using logic in the `datahub-gc` ingestion source.
+
+- Bug Fixes
+  - [UI] Fix a bug in displaying the filter value counts when selecting filters on the primary search experience
+  - [UI] Fix unnecessary horizontal scrolling wide markdown documentation.
+  - [UI] Fix bug in siblings external URLs. Now showing both the dbt and Snowflake URL as separate, correct URLs.
+  - [UI] Fix bug on listing data product assets with View applied
+  - [UI] Fix siblings bug in Schema Field queries tab
+  - [UI] Handle edge cycles in lineage graph more correctly
+  - [UI] Hide incorrect "Lineage" sidebar section on sibling pages (incorrect merge)
+  - [UI] Miscellaneous fixes to **Automations** forms UI - creating and editing automations.
+  - [UI] Fix scrolling to the end of the list of tabs on Asset Profiles
+  - [UI] Fix Compact View preview on Hover Card (looked squished!)
+  - [UI] Update asset counts on Domain profile pages after adding and removing assets right away
+  - [UI] Improve support for Compliance Forms and Structured Properties on sibling asset profile pages
+  - [Automations] **Column Description Propagation**: Fix Column Description Propagation issue where column description would not propagate if self-lineage was stored in graph index
+  - [Automations] **Snowflake Tag Sync**: Fix bug in Snowflake Tag Sync that failed to sync to columns with special characters
+
+- Product
+    - [BETA] Introducing the **BigQuery Metadata Sync Automation** to sync tags, glossary terms, and descriptions from DataHub to BigQuery. Check out the [feature guide](https://datahubproject.io/docs/automations/bigquery-metadata-sync/) for more information. To enable this BETA feature, reach out to your Acryl representative.
+    - [BETA] Introducing the **AI Classification Automation** to automatically classify your tables & columns using your organization's custom glossary terms. Check out the [feature guide](https://datahubproject.io/docs/automations/ai-term-suggestion) for more information. To enable this BETA feature, reach out to your Acryl representative.
+    - [BETA] A new way to visualize Column-Level Lineage, focused on a single column. Accessible by clicking on a column name in the column details sidebar or by clicking on the "Explore complete column lineage" button on a column in the regular lineage visualization. This will allow you to view only the upstreams and downstreams of the specific column being viewed. Please reach out to your Acryl representative to enable this feature.
+    - [BETA] Support running Automations via a Remote Executor using an Executor ID. This is currently in Beta, please reach out to your Acryl representative for more information.
+    - [BETA] Support plugging in custom Mixpanel or Google Analytics Measurement ID (GA4) to DataHub. Reach out to your Acryl representative for more information.
+    - Introducing **Structured Properties** UI. Create and manage custom properties for all asset types via the DataHub UI under **Govern** > **Structured Properties**. Feature guide will be coming in v0.3.8 - reach out to your Acryl representative for more information. Requires the `Manage Structured Properties` privilege to edit, `View Structured Properties` privilege to view.
+    - Introducing **Compliance Forms** UI. Create and manage compliance forms to run large-scale metadata collection initiatives inside your organization. Supported for all asset types via the DataHub UI under **Govern** > **Compliance Forms**. Feature guide will be coming in v0.3.8 - reach out to your Acryl representative for more information. Requires the `Manage Compliance Forms` privilege to edit, `View Compliance Forms` privilege to view. Compliance Forms also support analytics, which are updated once per day by default. 
+    - Support adding and removing structured properties from Table & Column Properties Tab
+    - Support filtering by Structured Properties in the search UI (main search only, not on lists yet)
+    - Acryl 2.0 is enabled by default for all users who have no explicitly set their display preference via **Settings** > **Appearance**. 
+    - Support searching the visible lineage graph by asset name 
+    - Support showing 'all' assets in a downstream or upstream lineage level in one click
+    - Support searching the assets hidden by a collapsed, "show more" node
+    - On lineage graph, draw an arrow from a column to the "show more" node if that column has lineage to a hidden node
+    - On lineage graph, add control to show lineage edges to entities that are deleted / do not exist
+    - Support deleting Data Product from the Data Product page 
+    - Support viewing & editing documentation in full-screen mode 
+    - Support copying queries for View Definitions (sidebar + tabs)
+    - Support V2 UI with Chrome Extension, fix miscellaneous bugs related to documentations, glossary, and lineage interactions. 
+    - Minor UX improvements (alignments, etc) to Quality, Assertions tabs. 
+    - Reorder the asset sidebar sections to prioritize documentation & lineage, the most used features. Moved down status, and share related tabs. 
+    - Add "Total Views" and "Recent Views" statistics to Dashboard & Chart asset sidebar header. 
+    - Ingestion UI: Always display the number of assets ingested on "Failure" & "Succeeded With Warnings"
+    - Permissions: Hide **Settings** > **Access Tokens** page if user doesn't have the `Generate Access Tokens` privilege.
+    - Add a Properties tab to Asset sidebar 
+    - Hide the 'notes' icon from the Columns table on Dataset Profiles, only show in the Column sidebar
+    - Add Properties Count, Column Count, Incident Count to Asset Profile tab names
+    - Allow resizing of the browse sizebar 
+    - Display custom Assertion Error messages via the UI
+    - Add sorting to Columns table 
+    - Add description to "hover preview" of assets 
+    - Rename 'Inbox' navigation item to 'Tasks' to align with rebranding as 'Task Center'
+    - Support viewing correctly merged schema change history for sibling pages
+    - Minor UX improvements on lineage graph 
+    - Minox UX improvements on Glossary, Search Cards, Home page, Subscriptions tab, and more.
+    - Improved usage-based search ranking. Please reach out with any questions or concerns
+    - Improved UX for setting up and managing SSO
+
+- Ingestion changes
+    - In addition to the improvements listed here: https://github.com/acryldata/datahub/releases/tag/v0.14.1.12
+    - PowerBI: Support for PowerBI Apps and cross-workspace lineage
+    - Fivetran: Major improvements to configurability and improved reliability with large Fivetran setups
+    - Snowflake & BigQuery: Improved handling of temporary tables and swap statements when generating lineage
+    - [Beta] Preset integration
+
+- Platform changes
+    - Added datahub-usage-reporting job to calculate usage metrics for search ranking     
+    - Metadata Test performance improvements: async ingestion & tag patch support 
+    - Authentication & RestAPI Authorization enabled by default
+    - Added datahub-gc and datahub-usage-reporting SYSTEM ingestion sources
+    - Added sweeper to executor to cancel duplicate and stale ingestion jobs
+    - Added soft delete status to edges in graph store
+    - Added service side options for newer clients with older service
+        - ALTERNATE_MCP_VALIDATION=true
+        - MCP_VALIDATION_IGNORE_UNKNOWN=true
+    - OpenAPIv3
+        - Added generic entities scroll endpoint
+        - Added `async` and `createIfNotExists` on aspect endpoints 
+    - System Operations privilege extended to all system operations
+    - [BETA] Introduce Entity Change Events Poll API behind permission "Get Platform Events". This enables programmatic access to entity change events in DataHub. Reach out to your Acryl representative for more information.
+    - (system / internal) Exclude form-prompt tests in live Metadata Tests evaluation
+    - (system / internal) Exclude form-prompt tests in stored Metadata Test results
+    - Elasticsearch reindex time limit of 8h removed
+    - Data Product Properties Unset side effect introduced
+      - Previously, Data Products could be set as linked to multiple Datasets if modified directly via the REST API rather than linked through the UI or GraphQL. This side effect aligns the REST API behavior with the GraphQL behavior by introducting a side effect that enforces the 1-to-1 constraint between Data Products and Datasets
+      - NOTE: There is a pathological pattern of writes for Data Products that can introduce issues with write processing that can occur with this side effect. If you are constantly changing all of the Datasets associated with a Data Product back and forth between multiple Data Products it will result in a high volume of writes due to the need to unset previous associations.
diff --git a/docs/managed-datahub/slack/saas-slack-setup.md b/docs/managed-datahub/slack/saas-slack-setup.md
index 590f57d2c2e70..9cc2ed6b96658 100644
--- a/docs/managed-datahub/slack/saas-slack-setup.md
+++ b/docs/managed-datahub/slack/saas-slack-setup.md
@@ -5,7 +5,7 @@ import FeatureAvailability from '@site/src/components/FeatureAvailability';
 <FeatureAvailability saasOnly />
 
 ## Install the DataHub Slack App into your Slack workspace
-
+You can see the permissions required by the DataHub Slack bot [below](#datahub-slack-bot-permissions).
 
 ### Video Walkthrough
 <div align="center"><iframe width="560" height="315" src="https://www.loom.com/embed/af6fcfd435cf4993b79a0c8e13aecaaa?sid=93f2a66b-1362-4809-996b-5abb399f82dd" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe></div>
@@ -29,7 +29,11 @@ The following steps should be performed by a Slack Workspace Admin.
 <p align="center">
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/slack/setup_4.png"/>
 </p>
-6. You will be automatically re-directed to Slack to confirm DataHub Slack App's permissions and complete the installation process:
+6. You will be automatically re-directed to Slack to confirm DataHub Slack App's permissions and complete the installation process.
+
+:::note
+You may need approval from a workspace admin to do this step. Learn about [what to do in this scenario here](#workspace-admin-approval-guide).
+:::
 <p align="center">
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/slack/setup_5.png"/>
 </p>
@@ -44,6 +48,50 @@ The following steps should be performed by a Slack Workspace Admin.
 
 Now proceed to the [Subscriptions and Notifications page](https://datahubproject.io/docs/managed-datahub/subscription-and-notification) to see how you can subscribe to be notified about events on the platform, or visit the [Slack App page](saas-slack-app.md) to see how you can use DataHub's powerful capabilities directly within Slack.
 
+### DataHub Slack bot permissions
+The DataHub Slack bot requires a certain set of scopes (permissions) to function. We've listed them below with thier explanations.
+```
+# Required for slash commands / shortcuts.
+commands
+# Required to get @DataHub messages and send messages as @DataHub.
+app_mentions:read
+chat:write
+chat:write.public
+# When sending messages we want to use a custom icon_url so that we can display the DataHub Cloud logo.
+chat:write.customize
+# Required to see conversation details + read messages.
+channels:history
+channels:read
+groups:history
+groups:read
+im:history
+im:read
+mpim:history
+mpim:read
+metadata.message:read
+# Required to get workspace ID and create links to user profiles.
+team:read
+# Allows the bot to join a public channel when someone configures notifications to be sent to one.
+channels:join
+# Required to unfurl links.
+links:read
+links:write
+# Required to resolve user IDs to names/emails + enable lookup by email address.
+users:read
+users:read.email
+# Future-proofing.
+reactions:read
+reactions:write
+```
+
+### Workspace admin approval guide
+In some workspaces, you will find at step 6 above you will need approval from your workspace admin. In this case, you will want to:
+1. On step #6 above, continue by clicking the button to request their approval.
+2. Once approved, you will get a notification from the **Slack bot** that your request has been approved. Follow the link it provides to complete the process.
+**If you do not get the message:** visit [api.slack.com/apps](https://api.slack.com/apps), open your DataHub app and complete the installation from there.
+3. Once it is done, you can visit `<your-datahub-instance-url>/settings/notifications`, and send a test notification to verify everything's working.
+4. **Making Slack commands work:** when running Slack commands such as `/datahub search my dataset` you may get a `dispatch_failed` error. To resolve this, see [the steps here](https://datahubproject.io/docs/managed-datahub/slack/saas-slack-troubleshoot#slack-bot-issues).
+
 <!-- NOTE: we are hiding this method for now. If a need arises, we can uncomment below docs.
 # Bot Token Method (Not recommended)
 
diff --git a/docs/managed-datahub/slack/saas-slack-troubleshoot.md b/docs/managed-datahub/slack/saas-slack-troubleshoot.md
index 2a2fa3ed4ed7f..92ae51c6ba50c 100644
--- a/docs/managed-datahub/slack/saas-slack-troubleshoot.md
+++ b/docs/managed-datahub/slack/saas-slack-troubleshoot.md
@@ -69,3 +69,24 @@ If you've subscribed to an entity, and then performed an action (i.e., raised an
 ### There is an issue with DataHub's systems
 If sending a test notification works, and you've verified that none of the above cases apply, then you should contact your DataHub Customer Success rep to help troubleshoot and resolve the issue. 
 
+## Slack bot issues
+Below you'll find some tips to troubleshoot issues with your Slack bot.
+
+### Command failed with error "dispatch_failed"
+If you've installed the Slack bot, but your commands are failing with an error 'dispatch_failed', you can try the following to correct it.
+1. Open your DataHub cloud instance with the following url: `<your-instance-base-url>/settings/integrations/slack?display_all_configs=true`.
+2. Switch to the **Bot Token** tab.
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/slack/verify_tokens_1.png"/>
+</p>
+3. The values will be `**`'d out, but if any of these fields are empty (ie. Signing Secret), then you'll want to fill them in.
+4. Visit [api.slack.com/apps](https://api.slack.com/apps), and open your currently installed app. You will see fields like `App ID` and `Signing Secret` here:
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/slack/verify_tokens_2.png"/>
+</p>
+5. You can get your `Bot Token` from the **OAuth & Permissions** tab in the side nav.
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/slack/verify_tokens_3.png"/>
+</p>
+6. Paste the values in and hit **Update Configuration**.
+7. Test the Slack command now and it should work. If it still fails, please reach out to your DataHub Cloud admin to troubleshoot further.
diff --git a/docs/plugins.md b/docs/plugins.md
index 12c192b5b5190..7212c74dad3c0 100644
--- a/docs/plugins.md
+++ b/docs/plugins.md
@@ -65,14 +65,14 @@ The sample authenticator implementation can be found at [Authenticator Sample](.
 3. Use `getResourceAsStream` to read files: If your plugin read any configuration file like properties or YAML or JSON or xml then use `this.getClass().getClassLoader().getResourceAsStream("<file-name>")` to read that file from DataHub GMS plugin's class-path. For DataHub GMS resource look-up behavior please refer [Plugin Installation](#plugin-installation) section. Sample code of `getResourceAsStream` is available in sample Authenticator plugin [TestAuthenticator.java](../metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthenticator.java).
 
 
-4. Bundle your Jar: Use `com.github.johnrengelman.shadow` gradle plugin to create an uber jar. 
+4. Bundle your Jar: Use `com.gradleup.shadow` gradle plugin to create an uber jar. 
 
    To see an example of building an uber jar, check out the `build.gradle` file for the apache-ranger-plugin file of [Apache Ranger Plugin](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin) for reference. 
 
    Exclude signature files as shown in below `shadowJar` task.
 
    ```groovy
-     apply plugin: 'com.github.johnrengelman.shadow';
+     apply plugin: 'com.gradleup.shadow';
      shadowJar {
          // Exclude com.datahub.plugins package and files related to jar signature   
          exclude "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA"
@@ -152,14 +152,14 @@ The sample authorizer implementation can be found at [Authorizer Sample](https:/
 
 3. Use `getResourceAsStream` to read files: If your plugin read any configuration file like properties or YAML or JSON or xml then use `this.getClass().getClassLoader().getResourceAsStream("<file-name>")` to read that file from DataHub GMS plugin's class-path. For DataHub GMS resource look-up behavior please refer [Plugin Installation](#plugin-installation) section. Sample code of `getResourceAsStream` is available in sample Authenticator plugin [TestAuthenticator.java](../metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthenticator.java).
 
-4. Bundle your Jar: Use `com.github.johnrengelman.shadow` gradle plugin to create an uber jar. 
+4. Bundle your Jar: Use `com.gradleup.shadow` gradle plugin to create an uber jar. 
 
    To see an example of building an uber jar, check out the `build.gradle` file for the apache-ranger-plugin file of [Apache Ranger Plugin](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin) for reference.
 
    Exclude signature files as shown in below `shadowJar` task.
 
    ```groovy
-     apply plugin: 'com.github.johnrengelman.shadow';
+     apply plugin: 'com.gradleup.shadow';
      shadowJar {
          // Exclude com.datahub.plugins package and files related to jar signature   
          exclude "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA"
diff --git a/docs/quick-ingestion-guides/bigquery/setup.md b/docs/quick-ingestion-guides/bigquery/setup.md
index 96850f2deb68e..7144ad6403ce9 100644
--- a/docs/quick-ingestion-guides/bigquery/setup.md
+++ b/docs/quick-ingestion-guides/bigquery/setup.md
@@ -30,7 +30,7 @@ Please refer to the BigQuery [Permissions](https://cloud.google.com/iam/docs/per
    * [BigQuery Job User](https://cloud.google.com/bigquery/docs/access-control#bigquery.jobUser)
    * [BigQuery Metadata Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer)
    * [BigQuery Resource Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.resourceViewer) -> This role is for Table-Level Lineage and Usage extraction
-   * [Logs View Accessor](https://cloud.google.com/bigquery/docs/access-control#bigquery.dataViewer) -> This role is for Table-Level Lineage and Usage extraction
+   * [Logs View Accessor](https://cloud.google.com/logging/docs/access-control#logging.viewAccessor) -> This role is for Table-Level Lineage and Usage extraction
    * [BigQuery Data Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.dataViewer) -> This role is for Profiling
    * [BigQuery Read Session User](https://cloud.google.com/bigquery/docs/access-control#bigquery.readSessionUser) -> This role is for Profiling
 
diff --git a/docs/quick-ingestion-guides/snowflake/setup.md b/docs/quick-ingestion-guides/snowflake/setup.md
index af5f15492376b..aaa9c67014814 100644
--- a/docs/quick-ingestion-guides/snowflake/setup.md
+++ b/docs/quick-ingestion-guides/snowflake/setup.md
@@ -43,6 +43,8 @@ In order to configure ingestion from Snowflake, you'll first have to ensure you
    grant select on future external tables in database identifier($db_var) to role datahub_role;
    grant select on all views in database identifier($db_var) to role datahub_role;
    grant select on future views in database identifier($db_var) to role datahub_role;
+   grant select on all dynamic tables in database identifier($db_var) to role datahub_role;
+   grant select on future dynamic tables in database identifier($db_var) to role datahub_role;
 
    --  Grant access to view tables and views
    grant references on all tables in database identifier($db_var) to role datahub_role;
@@ -51,6 +53,9 @@ In order to configure ingestion from Snowflake, you'll first have to ensure you
    grant references on future external tables in database identifier($db_var) to role datahub_role;
    grant references on all views in database identifier($db_var) to role datahub_role;
    grant references on future views in database identifier($db_var) to role datahub_role;
+   --  Grant access to dynamic tables
+   grant monitor on all dynamic tables in database identifier($db_var) to role datahub_role;
+   grant monitor on future dynamic tables in database identifier($db_var) to role datahub_role;
 
    -- Assign privileges to extract lineage and usage statistics from Snowflake by executing the below query.
    grant imported privileges on database snowflake to role datahub_role;
diff --git a/docs/what/relationship.md b/docs/what/relationship.md
index d5348dc04b3c0..13730827f178b 100644
--- a/docs/what/relationship.md
+++ b/docs/what/relationship.md
@@ -1,6 +1,6 @@
 # What is a relationship?
 
-A relationship is a named associate between exactly two [entities](entity.md), a source and a destination. 
+A relationship is a named associate between exactly two [entities](entity.md), a source and a destination.
 
 
 <p align="center">
@@ -8,100 +8,69 @@ A relationship is a named associate between exactly two [entities](entity.md), a
 </p>
 
 
-From the above graph, a `Group` entity can be linked to a `User` entity via a `HasMember` relationship. 
-Note that the name of the relationship reflects the direction, i.e. pointing from `Group` to `User`. 
-This is due to the fact that the actual metadata aspect holding this information is associated with `Group`, rather than User. 
-Had the direction been reversed, the relationship would have been named `IsMemberOf` instead. 
-See [Direction of Relationships](#direction-of-relationships) for more discussions on relationship directionality. 
-A specific instance of a relationship, e.g. `urn:li:corpGroup:group1` has a member `urn:li:corpuser:user1`, 
+From the above graph, a `Group` entity can be linked to a `User` entity via a `HasMember` relationship.
+Note that the name of the relationship reflects the direction, i.e. pointing from `Group` to `User`.
+This is due to the fact that the actual metadata aspect holding this information is associated with `Group`, rather than
+User.
+Had the direction been reversed, the relationship would have been named `IsMemberOf` instead.
+See [Direction of Relationships](#direction-of-relationships) for more discussions on relationship directionality.
+A specific instance of a relationship, e.g. `urn:li:corpGroup:group1` has a member `urn:li:corpuser:user1`,
 corresponds to an edge in the metadata graph.
 
-Similar to an entity, a relationship can also be associated with optional attributes that are derived from the metadata. 
-For example, from the `Membership` metadata aspect shown below, we’re able to derive the `HasMember` relationship that links a specific `Group` to a specific `User`. We can also include additional attribute to the relationship, e.g. importance, which corresponds to the position of the specific member in the original membership array. This allows complex graph query that travel only relationships that match certain criteria, e.g. "returns only the top-5 most important members of this group." 
-Similar to the entity attributes, relationship attributes should only be added based on the expected query patterns to reduce the indexing cost.
+Relationships are meant to be "entity-neutral". In other words, one would expect to use the same `OwnedBy` relationship
+to link a `Dataset` to a `User` and to link a `Dashboard` to a `User`.
+As Pegasus doesn’t allow typing a field using multiple URNs (because they’re all essentially strings), we resort to
+using generic URN type for the source and destination.
+We also introduce a `@Relationship` [annotation](../modeling/extending-the-metadata-model.md/#relationship) to
+limit the allowed source and destination URN types.
 
-```
-namespace: com.linkedin.group
-
-import com.linkedin.common.AuditStamp
-import com.linkedin.common.CorpuserUrn
-
-/**
- * The membership metadata for a group
- */
-record Membership {
-
-  /** Audit stamp for the last change */
-  modified: AuditStamp
-
-  /** Admin of the group */
-  admin: CorpuserUrn
-
-  /** Members of the group, ordered in descending importance */
-  members: array[CorpuserUrn]
-}
-```
-
-Relationships are meant to be "entity-neutral". In other words, one would expect to use the same `OwnedBy` relationship to link a `Dataset` to a `User` and to link a `Dashboard` to a `User`. As Pegasus doesn’t allow typing a field using multiple URNs (because they’re all essentially strings), we resort to using generic URN type for the source and destination. 
-We also introduce a `@pairings` [annotation](https://linkedin.github.io/rest.li/pdl_migration#shorthand-for-custom-properties) to limit the allowed source and destination URN types.
-
-While it’s possible to model relationships in rest.li as [association resources](https://linkedin.github.io/rest.li/modeling/modeling#association), which often get stored as mapping tables, it is far more common to model them as "foreign keys" field in a metadata aspect. For instance, the `Ownership` aspect is likely to contain an array of owner’s corpuser URNs.
+While it’s possible to model relationships in rest.li
+as [association resources](https://linkedin.github.io/rest.li/modeling/modeling#association), which often get stored as
+mapping tables, it is far more common to model them as "foreign keys" field in a metadata aspect. For instance,
+the `Ownership` aspect is likely to contain an array of owner’s corpUser URNs.
 
 Below is an example of how a relationship is modeled in PDL. Note that:
-1. As the `source` and `destination` are of generic URN type, we’re able to factor them out to a common `BaseRelationship` model.
-2. Each model is expected to have a `@pairings` annotation that is an array of all allowed source-destination URN pairs.
-3. Unlike entity attributes, there’s no requirement on making all relationship attributes optional since relationships do not support partial updates.
+
+1. This aspect, `nativeGroupMembership` would be associated with a `corpUser`
+2. The `corpUser`'s aspect points to one or more parent entities of type `corpGroup`
 
 ```
-namespace com.linkedin.metadata.relationship
+namespace com.linkedin.identity
 
 import com.linkedin.common.Urn
 
 /**
- * Common fields that apply to all relationships
+ * Carries information about the native CorpGroups a user is in.
  */
-record BaseRelationship {
-
-  /**
-   * Urn for the source of the relationship
-   */
-  source: Urn
-
-  /**
-   * Urn for the destination of the relationship
-   */
-  destination: Urn
+@Aspect = {
+  "name": "nativeGroupMembership"
 }
-```
-
-```
-namespace com.linkedin.metadata.relationship
-
-/**
- * Data model for a has-member relationship
- */
-@pairings = [ {
-  "destination" : "com.linkedin.common.urn.CorpGroupUrn",
-  "source" : "com.linkedin.common.urn.CorpUserUrn"
-} ]
-record HasMembership includes BaseRelationship
-{
-  /**
-   * The importance of the membership
-   */
-  importance: int 
+record NativeGroupMembership {
+  @Relationship = {
+    "/*": {
+      "name": "IsMemberOfNativeGroup",
+      "entityTypes": [ "corpGroup" ]
+    }
+  }
+  nativeGroups: array[Urn]
 }
 ```
 
 ## Direction of Relationships
 
-As relationships are modeled as directed edges between nodes, it’s natural to ask which way should it be pointing, 
-or should there be edges going both ways? The answer is, "doesn’t really matter." It’s rather an aesthetic choice than technical one. 
+As relationships are modeled as directed edges between nodes, it’s natural to ask which way should it be pointing,
+or should there be edges going both ways? The answer is, "doesn’t really matter." It’s rather an aesthetic choice than
+technical one.
 
-For one, the actual direction doesn’t really impact the execution of graph queries. Most graph DBs are fully capable of traversing edges in reverse direction efficiently.
+For one, the actual direction doesn’t really impact the execution of graph queries. Most graph DBs are fully capable of
+traversing edges in reverse direction efficiently.
 
-That being said, generally there’s a more "natural way" to specify the direction of a relationship, which closely relate to how the metadata is stored. For example, the membership information for an LDAP group is generally stored as a list in group’s metadata. As a result, it’s more natural to model a `HasMember` relationship that points from a group to a member, instead of a `IsMemberOf` relationship pointing from member to group.
+That being said, generally there’s a more "natural way" to specify the direction of a relationship, which closely relate
+to how the metadata is stored. For example, the membership information for an LDAP group is generally stored as a list
+in group’s metadata. As a result, it’s more natural to model a `HasMember` relationship that points from a group to a
+member, instead of a `IsMemberOf` relationship pointing from member to group.
 
 ## High Cardinality Relationships
 
-See [this doc](../advanced/high-cardinality.md) for suggestions on how to best model relationships with high cardinality.
+See [this doc](../advanced/high-cardinality.md) for suggestions on how to best model relationships with high
+cardinality.
diff --git a/docs/what/urn.md b/docs/what/urn.md
index 122b93038d9de..c7fb0555cd992 100644
--- a/docs/what/urn.md
+++ b/docs/what/urn.md
@@ -35,10 +35,17 @@ urn:li:dataset:(urn:li:dataPlatform:hdfs,PageViewEvent,EI)
 
 ## Restrictions
 
-There are a few restrictions when creating an urn:
+There are a few restrictions when creating an URN:
 
-1. Commas are reserved character in URN fields: `,`
-2. Parentheses are reserved characters in URN fields: `( , )`
-3. Colons are reserved characters in URN fields: `:`
+The following characters are not allowed anywhere in the URN
 
-Please do not use these characters when creating or generating urns.
+1. Parentheses are reserved characters in URN fields: `(` or `)`
+2. The "unit separator" unicode character `␟` (U+241F)
+
+The following characters are not allowed within an URN tuple.
+
+1. Commas are reserved characters in URN tuples: `,`
+
+Example: `urn:li:dashboard:(looker,dashboards.thelook)` is a valid urn, but `urn:li:dashboard:(looker,dashboards.the,look)` is invalid.
+
+Please do not use these characters when creating or generating urns. One approach is to use URL encoding for the characters.
diff --git a/entity-registry/build.gradle b/entity-registry/build.gradle
index 2dedea1f16d99..ee5ece4049399 100644
--- a/entity-registry/build.gradle
+++ b/entity-registry/build.gradle
@@ -4,9 +4,11 @@ plugins {
   id 'java-test-fixtures'
 }
 
+apply from: "../gradle/coverage/java-coverage.gradle"
+
 dependencies {
   implementation spec.product.pegasus.data
-  implementation spec.product.pegasus.generator
+  compileOnly spec.product.pegasus.generator
   api project(path: ':metadata-models')
   api project(path: ':metadata-models', configuration: "dataTemplate")
   api externalDependency.classGraph
@@ -25,6 +27,8 @@ dependencies {
       because("previous versions are vulnerable to CVE-2022-25857")
     }
   }
+  api project(path: ':li-utils')
+  api project(path: ':li-utils', configuration: "dataTemplate")
   dataModel project(':li-utils')
   annotationProcessor externalDependency.lombok
 
@@ -51,3 +55,4 @@ dependencies {
   testFixturesAnnotationProcessor externalDependency.lombok
 }
 compileTestJava.dependsOn tasks.getByPath(':entity-registry:custom-test-model:modelDeploy')
+
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/CachingAspectRetriever.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/CachingAspectRetriever.java
index 77e799f752455..375dd8cf8911e 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/CachingAspectRetriever.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/CachingAspectRetriever.java
@@ -1,4 +1,38 @@
 package com.linkedin.metadata.aspect;
 
+import com.linkedin.common.urn.Urn;
+import com.linkedin.entity.Aspect;
+import com.linkedin.metadata.models.registry.EmptyEntityRegistry;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nonnull;
+
 /** Responses can be cached based on application.yaml caching configuration for the EntityClient */
-public interface CachingAspectRetriever extends AspectRetriever {}
+public interface CachingAspectRetriever extends AspectRetriever {
+
+  CachingAspectRetriever EMPTY = new EmptyAspectRetriever();
+
+  class EmptyAspectRetriever implements CachingAspectRetriever {
+    @Nonnull
+    @Override
+    public Map<Urn, Map<String, Aspect>> getLatestAspectObjects(
+        Set<Urn> urns, Set<String> aspectNames) {
+      return Collections.emptyMap();
+    }
+
+    @Nonnull
+    @Override
+    public Map<Urn, Map<String, SystemAspect>> getLatestSystemAspects(
+        Map<Urn, Set<String>> urnAspectNames) {
+      return Collections.emptyMap();
+    }
+
+    @Nonnull
+    @Override
+    public EntityRegistry getEntityRegistry() {
+      return EmptyEntityRegistry.EMPTY;
+    }
+  }
+}
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/GraphRetriever.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/GraphRetriever.java
index f6858e7da4ba6..30a2c1eb9df8c 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/GraphRetriever.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/GraphRetriever.java
@@ -4,6 +4,7 @@
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.query.filter.RelationshipFilter;
 import com.linkedin.metadata.query.filter.SortCriterion;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Function;
 import javax.annotation.Nonnull;
@@ -97,4 +98,26 @@ default void consumeRelatedEntities(
       }
     }
   }
+
+  GraphRetriever EMPTY = new EmptyGraphRetriever();
+
+  class EmptyGraphRetriever implements GraphRetriever {
+
+    @Nonnull
+    @Override
+    public RelatedEntitiesScrollResult scrollRelatedEntities(
+        @Nullable List<String> sourceTypes,
+        @Nonnull Filter sourceEntityFilter,
+        @Nullable List<String> destinationTypes,
+        @Nonnull Filter destinationEntityFilter,
+        @Nonnull List<String> relationshipTypes,
+        @Nonnull RelationshipFilter relationshipFilter,
+        @Nonnull List<SortCriterion> sortCriterion,
+        @Nullable String scrollId,
+        int count,
+        @Nullable Long startTimeMillis,
+        @Nullable Long endTimeMillis) {
+      return new RelatedEntitiesScrollResult(0, 0, null, Collections.emptyList());
+    }
+  }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java
index dc7934ad5cc19..6ce6a9a573038 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java
@@ -9,6 +9,7 @@
 import com.linkedin.util.Pair;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -27,10 +28,12 @@
 public interface AspectsBatch {
   Collection<? extends BatchItem> getItems();
 
+  Collection<? extends BatchItem> getInitialItems();
+
   RetrieverContext getRetrieverContext();
 
   /**
-   * Returns MCP items. Could be patch, upsert, etc.
+   * Returns MCP items. Could be one of patch, upsert, etc.
    *
    * @return batch items
    */
@@ -49,7 +52,8 @@ default List<MCPItem> getMCPItems() {
    *     various hooks
    */
   Pair<Map<String, Set<String>>, List<ChangeMCP>> toUpsertBatchItems(
-      Map<String, Map<String, SystemAspect>> latestAspects);
+      Map<String, Map<String, SystemAspect>> latestAspects,
+      Map<String, Map<String, Long>> nextVersions);
 
   /**
    * Apply read mutations to batch
@@ -158,13 +162,24 @@ static Stream<MCLItem> applyMCLSideEffects(
   }
 
   default boolean containsDuplicateAspects() {
-    return getItems().stream()
-            .map(i -> String.format("%s_%s", i.getClass().getName(), i.hashCode()))
+    return getInitialItems().stream()
+            .map(i -> String.format("%s_%s", i.getClass().getSimpleName(), i.hashCode()))
             .distinct()
             .count()
         != getItems().size();
   }
 
+  default Map<String, List<? extends BatchItem>> duplicateAspects() {
+    return getInitialItems().stream()
+        .collect(
+            Collectors.groupingBy(
+                i -> String.format("%s_%s", i.getClass().getSimpleName(), i.hashCode())))
+        .entrySet()
+        .stream()
+        .filter(entry -> entry.getValue() != null && entry.getValue().size() > 1)
+        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+  }
+
   default Map<String, Set<String>> getUrnAspectsMap() {
     return getItems().stream()
         .map(aspect -> Pair.of(aspect.getUrn().toString(), aspect.getAspectName()))
@@ -227,4 +242,39 @@ static String toAbbreviatedString(Collection<? extends BatchItem> items, int max
         + StringUtils.abbreviate(itemsAbbreviated.toString(), maxWidth)
         + '}';
   }
+
+  /**
+   * Increment aspect within a batch, tracking both the next aspect version and the most recent
+   *
+   * @param changeMCP changeMCP to be incremented
+   * @param latestAspects lastest aspects within the batch
+   * @param nextVersions next version for the aspects in the batch
+   * @return the incremented changeMCP
+   */
+  static ChangeMCP incrementBatchVersion(
+      ChangeMCP changeMCP,
+      Map<String, Map<String, SystemAspect>> latestAspects,
+      Map<String, Map<String, Long>> nextVersions) {
+    long nextVersion =
+        nextVersions
+            .getOrDefault(changeMCP.getUrn().toString(), Collections.emptyMap())
+            .getOrDefault(changeMCP.getAspectName(), 0L);
+
+    changeMCP.setPreviousSystemAspect(
+        latestAspects
+            .getOrDefault(changeMCP.getUrn().toString(), Collections.emptyMap())
+            .getOrDefault(changeMCP.getAspectName(), null));
+
+    changeMCP.setNextAspectVersion(nextVersion);
+
+    // support inner-batch upserts
+    latestAspects
+        .computeIfAbsent(changeMCP.getUrn().toString(), key -> new HashMap<>())
+        .put(changeMCP.getAspectName(), changeMCP.getSystemAspect(nextVersion));
+    nextVersions
+        .computeIfAbsent(changeMCP.getUrn().toString(), key -> new HashMap<>())
+        .put(changeMCP.getAspectName(), nextVersion + 1);
+
+    return changeMCP;
+  }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java
index a6dfbc277e12e..7f0a849a0eda1 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java
@@ -23,4 +23,11 @@ public interface BatchItem extends ReadItem {
    */
   @Nonnull
   ChangeType getChangeType();
+
+  /**
+   * Determines if this item is a duplicate of another item in terms of the operation it represents
+   * to the database.Each implementation can define what constitutes a duplicate based on its
+   * specific fields which are persisted.
+   */
+  boolean isDatabaseDuplicateOf(BatchItem other);
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java
index e4143851afbe5..b78d563147e63 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java
@@ -17,10 +17,16 @@ public class CustomPropertiesPatchBuilder<T extends AbstractMultiFieldPatchBuild
   public static final String CUSTOM_PROPERTIES_BASE_PATH = "/customProperties";
 
   private final T parent;
-  private final List<ImmutableTriple<String, String, JsonNode>> operations = new ArrayList<>();
+  private final List<ImmutableTriple<String, String, JsonNode>> operations;
 
   public CustomPropertiesPatchBuilder(T parentBuilder) {
     this.parent = parentBuilder;
+    if (parentBuilder != null) {
+      // If a parent builder is provided, we use the same path operations list.
+      this.operations = parentBuilder.getPathValues();
+    } else {
+      this.operations = new ArrayList<>();
+    }
   }
 
   /**
@@ -72,9 +78,4 @@ public CustomPropertiesPatchBuilder<T> setProperties(Map<String, String> propert
   public T getParent() {
     return parent;
   }
-
-  @Override
-  public List<ImmutableTriple<String, String, JsonNode>> getSubPaths() {
-    return operations;
-  }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java
index 6a114d90875fe..231956a2fcec8 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java
@@ -4,12 +4,10 @@
 import static com.linkedin.metadata.Constants.DATA_FLOW_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.DATA_FLOW_INFO_ASPECT_NAME;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.TimeStamp;
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport;
-import java.util.List;
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -87,28 +85,23 @@ public DataFlowInfoPatchBuilder setCreated(@Nullable TimeStamp created) {
   }
 
   public DataFlowInfoPatchBuilder setLastModified(@Nullable TimeStamp lastModified) {
+    ObjectNode lastModifiedNode = instance.objectNode();
     if (lastModified == null) {
       pathValues.add(
           ImmutableTriple.of(
               PatchOperationType.REMOVE.getValue(), BASE_PATH + LAST_MODIFIED_KEY, null));
+    } else {
+      lastModifiedNode.put(TIME_KEY, lastModified.getTime());
+      if (lastModified.getActor() != null) {
+        lastModifiedNode.put(ACTOR_KEY, lastModified.getActor().toString());
+      }
+      pathValues.add(
+          ImmutableTriple.of(
+              PatchOperationType.ADD.getValue(), BASE_PATH + LAST_MODIFIED_KEY, lastModifiedNode));
     }
-    ObjectNode lastModifiedNode = instance.objectNode();
-    lastModifiedNode.put(TIME_KEY, lastModified.getTime());
-    if (lastModified.getActor() != null) {
-      lastModifiedNode.put(ACTOR_KEY, lastModified.getActor().toString());
-    }
-    pathValues.add(
-        ImmutableTriple.of(
-            PatchOperationType.ADD.getValue(), BASE_PATH + LAST_MODIFIED_KEY, lastModifiedNode));
     return this;
   }
 
-  @Override
-  protected List<ImmutableTriple<String, String, JsonNode>> getPathValues() {
-    pathValues.addAll(customPropertiesPatchBuilder.getSubPaths());
-    return pathValues;
-  }
-
   @Override
   protected String getAspectName() {
     return DATA_FLOW_INFO_ASPECT_NAME;
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java
index 99c0ac6c15eb1..dd17fbacf338e 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java
@@ -4,13 +4,11 @@
 import static com.linkedin.metadata.Constants.DATA_JOB_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.DATA_JOB_INFO_ASPECT_NAME;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.TimeStamp;
 import com.linkedin.common.urn.DataFlowUrn;
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport;
-import java.util.List;
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -113,12 +111,6 @@ public DataJobInfoPatchBuilder setLastModified(@Nullable TimeStamp lastModified)
     return this;
   }
 
-  @Override
-  protected List<ImmutableTriple<String, String, JsonNode>> getPathValues() {
-    pathValues.addAll(customPropertiesPatchBuilder.getSubPaths());
-    return pathValues;
-  }
-
   @Override
   protected String getAspectName() {
     return DATA_JOB_INFO_ASPECT_NAME;
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilder.java
index 6fffb17521ddb..14fc92a1bf3c8 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilder.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilder.java
@@ -15,6 +15,8 @@
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.graph.LineageDirection;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutableTriple;
 
 public class DataJobInputOutputPatchBuilder
@@ -24,6 +26,7 @@ public class DataJobInputOutputPatchBuilder
   private static final String OUTPUT_DATASET_EDGES_PATH_START = "/outputDatasetEdges/";
   private static final String INPUT_DATASET_FIELDS_PATH_START = "/inputDatasetFields/";
   private static final String OUTPUT_DATASET_FIELDS_PATH_START = "/outputDatasetFields/";
+  private static final String FINE_GRAINED_PATH_START = "/fineGrainedLineages/";
 
   // Simplified with just Urn
   public DataJobInputOutputPatchBuilder addInputDatajobEdge(@Nonnull DataJobUrn dataJobUrn) {
@@ -136,6 +139,103 @@ public DataJobInputOutputPatchBuilder addEdge(
     return this;
   }
 
+  /**
+   * Adds a field as a fine grained upstream
+   *
+   * @param upstreamSchemaField a schema field to be marked as upstream, format:
+   *     urn:li:schemaField(DATASET_URN, COLUMN NAME)
+   * @param confidenceScore optional, confidence score for the lineage edge. Defaults to 1.0 for
+   *     full confidence
+   * @param transformationOperation string operation type that describes the transformation
+   *     operation happening in the lineage edge
+   * @param downstreamSchemaField the downstream schema field this upstream is derived from, format:
+   *     urn:li:schemaField(DATASET_URN, COLUMN NAME)
+   * @param queryUrn query urn the relationship is derived from
+   * @return this builder
+   */
+  public DataJobInputOutputPatchBuilder addFineGrainedUpstreamField(
+      @Nonnull Urn upstreamSchemaField,
+      @Nullable Float confidenceScore,
+      @Nonnull String transformationOperation,
+      @Nonnull Urn downstreamSchemaField,
+      @Nullable Urn queryUrn) {
+    Float finalConfidenceScore = getConfidenceScoreOrDefault(confidenceScore);
+    String finalQueryUrn;
+    if (queryUrn == null || StringUtils.isBlank(queryUrn.toString())) {
+      finalQueryUrn = "NONE";
+    } else {
+      finalQueryUrn = queryUrn.toString();
+    }
+
+    ObjectNode fineGrainedLineageNode = instance.objectNode();
+    fineGrainedLineageNode.put("confidenceScore", instance.numberNode(finalConfidenceScore));
+    pathValues.add(
+        ImmutableTriple.of(
+            PatchOperationType.ADD.getValue(),
+            FINE_GRAINED_PATH_START
+                + transformationOperation
+                + "/"
+                + encodeValueUrn(downstreamSchemaField)
+                + "/"
+                + finalQueryUrn
+                + "/"
+                + encodeValueUrn(upstreamSchemaField),
+            fineGrainedLineageNode));
+
+    return this;
+  }
+
+  private Float getConfidenceScoreOrDefault(@Nullable Float confidenceScore) {
+    float finalConfidenceScore;
+    if (confidenceScore != null && confidenceScore > 0 && confidenceScore <= 1.0f) {
+      finalConfidenceScore = confidenceScore;
+    } else {
+      finalConfidenceScore = 1.0f;
+    }
+
+    return finalConfidenceScore;
+  }
+
+  /**
+   * Removes a field as a fine grained upstream
+   *
+   * @param upstreamSchemaField a schema field to be marked as upstream, format:
+   *     urn:li:schemaField(DATASET_URN, COLUMN NAME)
+   * @param transformationOperation string operation type that describes the transformation
+   *     operation happening in the lineage edge
+   * @param downstreamSchemaField the downstream schema field this upstream is derived from, format:
+   *     urn:li:schemaField(DATASET_URN, COLUMN NAME)
+   * @param queryUrn query urn the relationship is derived from
+   * @return this builder
+   */
+  public DataJobInputOutputPatchBuilder removeFineGrainedUpstreamField(
+      @Nonnull Urn upstreamSchemaField,
+      @Nonnull String transformationOperation,
+      @Nonnull Urn downstreamSchemaField,
+      @Nullable Urn queryUrn) {
+
+    String finalQueryUrn;
+    if (queryUrn == null || StringUtils.isBlank(queryUrn.toString())) {
+      finalQueryUrn = "NONE";
+    } else {
+      finalQueryUrn = queryUrn.toString();
+    }
+    pathValues.add(
+        ImmutableTriple.of(
+            PatchOperationType.REMOVE.getValue(),
+            FINE_GRAINED_PATH_START
+                + transformationOperation
+                + "/"
+                + encodeValueUrn(downstreamSchemaField)
+                + "/"
+                + finalQueryUrn
+                + "/"
+                + encodeValueUrn(upstreamSchemaField),
+            null));
+
+    return this;
+  }
+
   public DataJobInputOutputPatchBuilder removeEdge(
       @Nonnull Edge edge, @Nonnull LineageDirection direction) {
     String path = getEdgePath(edge, direction);
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java
index 31e181fc244fb..60d52c7c72088 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java
@@ -4,10 +4,8 @@
 import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.DATASET_PROPERTIES_ASPECT_NAME;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport;
-import java.util.List;
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -116,12 +114,6 @@ public DatasetPropertiesPatchBuilder setCustomProperties(Map<String, String> pro
     return this;
   }
 
-  @Override
-  protected List<ImmutableTriple<String, String, JsonNode>> getPathValues() {
-    pathValues.addAll(customPropertiesPatchBuilder.getSubPaths());
-    return pathValues;
-  }
-
   @Override
   protected String getAspectName() {
     return DATASET_PROPERTIES_ASPECT_NAME;
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/UpstreamLineagePatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/UpstreamLineagePatchBuilder.java
index 08182761aeb03..d0a46a35d5182 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/UpstreamLineagePatchBuilder.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/UpstreamLineagePatchBuilder.java
@@ -142,7 +142,7 @@ public UpstreamLineagePatchBuilder removeFineGrainedUpstreamField(
             FINE_GRAINED_PATH_START
                 + transformationOperation
                 + "/"
-                + downstreamSchemaField
+                + encodeValueUrn(downstreamSchemaField)
                 + "/"
                 + finalQueryUrn
                 + "/"
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java
index d891a6b9673da..cd74818c24e19 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java
@@ -1,9 +1,6 @@
 package com.linkedin.metadata.aspect.patch.builder.subtypesupport;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.linkedin.metadata.aspect.patch.builder.AbstractMultiFieldPatchBuilder;
-import java.util.List;
-import org.apache.commons.lang3.tuple.ImmutableTriple;
 
 /**
  * Used for supporting intermediate subtypes when constructing a patch for an aspect that includes
@@ -15,10 +12,4 @@ public interface IntermediatePatchBuilder<T extends AbstractMultiFieldPatchBuild
 
   /** Convenience method to return parent patch builder in functional callstack */
   T getParent();
-
-  /**
-   * Exposes subpath values to parent patch builder in Op, Path, Value triples. Should usually only
-   * be called by the parent patch builder class when constructing the path values.
-   */
-  List<ImmutableTriple<String, String, JsonNode>> getSubPaths();
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/FineGrainedLineageTemplateHelper.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/FineGrainedLineageTemplateHelper.java
new file mode 100644
index 0000000000000..1f6a58c52ba24
--- /dev/null
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/FineGrainedLineageTemplateHelper.java
@@ -0,0 +1,282 @@
+package com.linkedin.metadata.aspect.patch.template;
+
+import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*;
+import static com.linkedin.metadata.Constants.*;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Streams;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.codehaus.plexus.util.StringUtils;
+
+public class FineGrainedLineageTemplateHelper {
+
+  private static final String FINE_GRAINED_UPSTREAM_TYPE = "upstreamType";
+  private static final String FINE_GRAINED_UPSTREAMS = "upstreams";
+  private static final String FINE_GRAINED_DOWNSTREAM_TYPE = "downstreamType";
+  private static final String FINE_GRAINED_DOWNSTREAMS = "downstreams";
+  private static final String FINE_GRAINED_TRANSFORMATION_OPERATION = "transformOperation";
+  private static final String FINE_GRAINED_CONFIDENCE_SCORE = "confidenceScore";
+  private static final String FINE_GRAINED_QUERY_ID = "query";
+
+  // Template support
+  private static final String NONE_TRANSFORMATION_TYPE = "NONE";
+  private static final Float DEFAULT_CONFIDENCE_SCORE = 1.0f;
+  private static final String DEFAULT_QUERY_ID = "NONE";
+
+  /**
+   * Combines fine grained lineage array into a map using upstream and downstream types as keys,
+   * defaulting when not present. Due to this construction, patches will look like: path:
+   * /fineGrainedLineages/TRANSFORMATION_OPERATION/DOWNSTREAM_FIELD_URN/QUERY_ID/UPSTREAM_FIELD_URN,
+   * op: ADD/REMOVE, value: float (confidenceScore) Due to the way FineGrainedLineage was designed
+   * it doesn't necessarily have a consistent key we can reference, so this specialized method
+   * mimics the arrayFieldToMap of the super class with the specialization that it does not put the
+   * full value of the aspect at the end of the key, just the particular array. This prevents
+   * unintended overwrites through improper MCP construction that is technically allowed by the
+   * schema when combining under fields that form the natural key.
+   *
+   * @param fineGrainedLineages the fine grained lineage array node
+   * @return the modified {@link JsonNode} with array fields transformed to maps
+   */
+  public static JsonNode combineAndTransformFineGrainedLineages(
+      @Nullable JsonNode fineGrainedLineages) {
+    ObjectNode mapNode = instance.objectNode();
+    if (!(fineGrainedLineages instanceof ArrayNode) || fineGrainedLineages.isEmpty()) {
+      return mapNode;
+    }
+    JsonNode lineageCopy = fineGrainedLineages.deepCopy();
+
+    lineageCopy
+        .elements()
+        .forEachRemaining(
+            node -> {
+              JsonNode nodeClone = node.deepCopy();
+              String transformationOperation =
+                  nodeClone.has(FINE_GRAINED_TRANSFORMATION_OPERATION)
+                      ? nodeClone.get(FINE_GRAINED_TRANSFORMATION_OPERATION).asText()
+                      : NONE_TRANSFORMATION_TYPE;
+
+              if (!mapNode.has(transformationOperation)) {
+                mapNode.set(transformationOperation, instance.objectNode());
+              }
+              ObjectNode transformationOperationNode =
+                  (ObjectNode) mapNode.get(transformationOperation);
+
+              ArrayNode downstreams =
+                  nodeClone.has(FINE_GRAINED_DOWNSTREAMS)
+                      ? (ArrayNode) nodeClone.get(FINE_GRAINED_DOWNSTREAMS)
+                      : null;
+
+              if (downstreams == null || downstreams.size() != 1) {
+                throw new UnsupportedOperationException(
+                    "Patching not supported on fine grained lineages with not"
+                        + " exactly one downstream. Current fine grained lineage implementation is downstream derived and "
+                        + "patches are keyed on the root of this derivation.");
+              }
+
+              Float confidenceScore =
+                  nodeClone.has(FINE_GRAINED_CONFIDENCE_SCORE)
+                      ? nodeClone.get(FINE_GRAINED_CONFIDENCE_SCORE).floatValue()
+                      : DEFAULT_CONFIDENCE_SCORE;
+
+              String upstreamType =
+                  nodeClone.has(FINE_GRAINED_UPSTREAM_TYPE)
+                      ? nodeClone.get(FINE_GRAINED_UPSTREAM_TYPE).asText()
+                      : null;
+              String downstreamType =
+                  nodeClone.has(FINE_GRAINED_DOWNSTREAM_TYPE)
+                      ? nodeClone.get(FINE_GRAINED_DOWNSTREAM_TYPE).asText()
+                      : null;
+              ArrayNode upstreams =
+                  nodeClone.has(FINE_GRAINED_UPSTREAMS)
+                      ? (ArrayNode) nodeClone.get(FINE_GRAINED_UPSTREAMS)
+                      : null;
+
+              String queryId =
+                  nodeClone.has(FINE_GRAINED_QUERY_ID)
+                      ? nodeClone.get(FINE_GRAINED_QUERY_ID).asText()
+                      : DEFAULT_QUERY_ID;
+
+              if (upstreamType == null) {
+                // Determine default type
+                Urn upstreamUrn =
+                    upstreams != null ? UrnUtils.getUrn(upstreams.get(0).asText()) : null;
+                if (upstreamUrn != null
+                    && DATASET_ENTITY_NAME.equals(upstreamUrn.getEntityType())) {
+                  upstreamType = FINE_GRAINED_LINEAGE_DATASET_TYPE;
+                } else {
+                  upstreamType = FINE_GRAINED_LINEAGE_FIELD_SET_TYPE;
+                }
+              }
+
+              if (downstreamType == null) {
+                // Always use FIELD type, only support patches for single field downstream
+                downstreamType = FINE_GRAINED_LINEAGE_FIELD_TYPE;
+              }
+
+              String downstreamRoot = downstreams.get(0).asText();
+              if (!transformationOperationNode.has(downstreamRoot)) {
+                transformationOperationNode.set(downstreamRoot, instance.objectNode());
+              }
+              ObjectNode downstreamRootNode =
+                  (ObjectNode) transformationOperationNode.get(downstreamRoot);
+              if (!downstreamRootNode.has(queryId)) {
+                downstreamRootNode.set(queryId, instance.objectNode());
+              }
+              ObjectNode queryNode = (ObjectNode) downstreamRootNode.get(queryId);
+              if (upstreams != null) {
+                addUrnsToParent(
+                    queryNode, upstreams, confidenceScore, upstreamType, downstreamType);
+              }
+            });
+    return mapNode;
+  }
+
+  private static void addUrnsToParent(
+      JsonNode parentNode,
+      ArrayNode urnsList,
+      Float confidenceScore,
+      String upstreamType,
+      String downstreamType) {
+    // Will overwrite repeat urns with different confidence scores with the most recently seen
+    ((ObjectNode) parentNode)
+        .setAll(
+            Streams.stream(urnsList.elements())
+                .map(JsonNode::asText)
+                .distinct()
+                .collect(
+                    Collectors.toMap(
+                        urn -> urn,
+                        urn ->
+                            mapToLineageValueNode(confidenceScore, upstreamType, downstreamType))));
+  }
+
+  private static JsonNode mapToLineageValueNode(
+      Float confidenceScore, String upstreamType, String downstreamType) {
+    ObjectNode objectNode = instance.objectNode();
+    objectNode.set(FINE_GRAINED_CONFIDENCE_SCORE, instance.numberNode(confidenceScore));
+    objectNode.set(FINE_GRAINED_UPSTREAM_TYPE, instance.textNode(upstreamType));
+    objectNode.set(FINE_GRAINED_DOWNSTREAM_TYPE, instance.textNode(downstreamType));
+    return objectNode;
+  }
+
+  /**
+   * Takes the transformed fine grained lineages map from pre-processing and reconstructs an array
+   * of FineGrainedLineages Avoids producing side effects by copying nodes, use resulting node and
+   * not the original
+   *
+   * @param transformedFineGrainedLineages the transformed fine grained lineage map
+   * @return the modified {@link JsonNode} formatted consistent with the original schema
+   */
+  public static ArrayNode reconstructFineGrainedLineages(JsonNode transformedFineGrainedLineages) {
+    if (transformedFineGrainedLineages instanceof ArrayNode) {
+      // We already have an ArrayNode, no need to transform. This happens during `replace`
+      // operations
+      return (ArrayNode) transformedFineGrainedLineages;
+    }
+    ObjectNode mapNode = (ObjectNode) transformedFineGrainedLineages;
+    ArrayNode fineGrainedLineages = instance.arrayNode();
+
+    mapNode
+        .fieldNames()
+        .forEachRemaining(
+            transformationOperation -> {
+              final ObjectNode transformationOperationNode =
+                  (ObjectNode) mapNode.get(transformationOperation);
+              transformationOperationNode
+                  .fieldNames()
+                  .forEachRemaining(
+                      downstreamName -> {
+                        final ObjectNode downstreamNode =
+                            (ObjectNode) transformationOperationNode.get(downstreamName);
+                        downstreamNode
+                            .fieldNames()
+                            .forEachRemaining(
+                                queryId ->
+                                    buildFineGrainedLineage(
+                                        downstreamName,
+                                        downstreamNode,
+                                        queryId,
+                                        transformationOperation,
+                                        fineGrainedLineages));
+                      });
+            });
+
+    return fineGrainedLineages;
+  }
+
+  private static void buildFineGrainedLineage(
+      final String downstreamName,
+      final ObjectNode downstreamNode,
+      final String queryId,
+      final String transformationOperation,
+      final ArrayNode fineGrainedLineages) {
+    final ObjectNode fineGrainedLineage = instance.objectNode();
+    final ObjectNode queryNode = (ObjectNode) downstreamNode.get(queryId);
+    if (queryNode.isEmpty()) {
+      // Short circuit if no upstreams left
+      return;
+    }
+    ArrayNode downstream = instance.arrayNode();
+    downstream.add(instance.textNode(downstreamName));
+    // Set defaults, if found in sub nodes override, for confidenceScore take lowest
+    AtomicReference<Float> minimumConfidenceScore = new AtomicReference<>(DEFAULT_CONFIDENCE_SCORE);
+    AtomicReference<String> upstreamType =
+        new AtomicReference<>(FINE_GRAINED_LINEAGE_FIELD_SET_TYPE);
+    AtomicReference<String> downstreamType = new AtomicReference<>(FINE_GRAINED_LINEAGE_FIELD_TYPE);
+    ArrayNode upstreams = instance.arrayNode();
+    queryNode
+        .fieldNames()
+        .forEachRemaining(
+            upstream ->
+                processUpstream(
+                    queryNode,
+                    upstream,
+                    minimumConfidenceScore,
+                    upstreamType,
+                    downstreamType,
+                    upstreams));
+    fineGrainedLineage.set(FINE_GRAINED_DOWNSTREAMS, downstream);
+    fineGrainedLineage.set(FINE_GRAINED_UPSTREAMS, upstreams);
+    if (StringUtils.isNotBlank(queryId) && !DEFAULT_QUERY_ID.equals(queryId)) {
+      fineGrainedLineage.set(FINE_GRAINED_QUERY_ID, instance.textNode(queryId));
+    }
+    fineGrainedLineage.set(FINE_GRAINED_UPSTREAM_TYPE, instance.textNode(upstreamType.get()));
+    fineGrainedLineage.set(FINE_GRAINED_DOWNSTREAM_TYPE, instance.textNode(downstreamType.get()));
+    fineGrainedLineage.set(
+        FINE_GRAINED_CONFIDENCE_SCORE, instance.numberNode(minimumConfidenceScore.get()));
+    fineGrainedLineage.set(
+        FINE_GRAINED_TRANSFORMATION_OPERATION, instance.textNode(transformationOperation));
+    fineGrainedLineages.add(fineGrainedLineage);
+  }
+
+  private static void processUpstream(
+      final ObjectNode queryNode,
+      final String upstream,
+      final AtomicReference<Float> minimumConfidenceScore,
+      final AtomicReference<String> upstreamType,
+      final AtomicReference<String> downstreamType,
+      final ArrayNode upstreams) {
+    final ObjectNode upstreamNode = (ObjectNode) queryNode.get(upstream);
+    if (upstreamNode.has(FINE_GRAINED_CONFIDENCE_SCORE)) {
+      Float scoreValue = upstreamNode.get(FINE_GRAINED_CONFIDENCE_SCORE).floatValue();
+      if (scoreValue <= minimumConfidenceScore.get()) {
+        minimumConfidenceScore.set(scoreValue);
+      }
+    }
+    // Set types to last encountered, should never change, but this at least tries to support
+    // other types being specified.
+    if (upstreamNode.has(FINE_GRAINED_UPSTREAM_TYPE)) {
+      upstreamType.set(upstreamNode.get(FINE_GRAINED_UPSTREAM_TYPE).asText());
+    }
+    if (upstreamNode.has(FINE_GRAINED_DOWNSTREAM_TYPE)) {
+      downstreamType.set(upstreamNode.get(FINE_GRAINED_DOWNSTREAM_TYPE).asText());
+    }
+    upstreams.add(instance.textNode(upstream));
+  }
+}
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java
index d4e94e1e82e8f..23879ad1c2e35 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java
@@ -81,14 +81,13 @@ public static JsonNode populateTopLevelKeys(JsonNode transformedNode, JsonPatch
           PatchOperationType.REMOVE.equals(operationPath.getFirst())
               ? keys.length
               : keys.length - 1;
-
       // Skip first as it will always be blank due to path starting with /
       for (int i = 1; i < endIdx; i++) {
-        if (parent.get(keys[i]) == null) {
-          String decodedKey = decodeValue(keys[i]);
+        String decodedKey = decodeValue(keys[i]);
+        if (parent.get(decodedKey) == null) {
           ((ObjectNode) parent).set(decodedKey, instance.objectNode());
         }
-        parent = parent.get(keys[i]);
+        parent = parent.get(decodedKey);
       }
     }
 
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInputOutputTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInputOutputTemplate.java
index 3d398d97b50c3..ef26eed2f814f 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInputOutputTemplate.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInputOutputTemplate.java
@@ -1,6 +1,10 @@
 package com.linkedin.metadata.aspect.patch.template.datajob;
 
+import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*;
+import static com.linkedin.metadata.Constants.*;
+
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.DataJobUrnArray;
 import com.linkedin.common.DatasetUrnArray;
 import com.linkedin.common.EdgeArray;
@@ -9,6 +13,7 @@
 import com.linkedin.datajob.DataJobInputOutput;
 import com.linkedin.dataset.FineGrainedLineageArray;
 import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate;
+import com.linkedin.metadata.aspect.patch.template.FineGrainedLineageTemplateHelper;
 import java.util.Collections;
 import javax.annotation.Nonnull;
 
@@ -23,6 +28,8 @@ public class DataJobInputOutputTemplate implements ArrayMergingTemplate<DataJobI
   private static final String INPUT_DATASET_FIELDS_FIELD_NAME = "inputDatasetFields";
   private static final String OUTPUT_DATASET_FIELDS_FIELD_NAME = "outputDatasetFields";
 
+  private static final String FINE_GRAINED_LINEAGES_FIELD_NAME = "fineGrainedLineages";
+
   @Override
   public DataJobInputOutput getSubtype(RecordTemplate recordTemplate) throws ClassCastException {
     if (recordTemplate instanceof DataJobInputOutput) {
@@ -81,6 +88,12 @@ public JsonNode transformFields(JsonNode baseNode) {
     transformedNode =
         arrayFieldToMap(transformedNode, OUTPUT_DATASET_FIELDS_FIELD_NAME, Collections.emptyList());
 
+    ((ObjectNode) transformedNode)
+        .set(
+            FINE_GRAINED_LINEAGES_FIELD_NAME,
+            FineGrainedLineageTemplateHelper.combineAndTransformFineGrainedLineages(
+                transformedNode.get(FINE_GRAINED_LINEAGES_FIELD_NAME)));
+
     return transformedNode;
   }
 
@@ -112,6 +125,12 @@ public JsonNode rebaseFields(JsonNode patched) {
         transformedMapToArray(
             rebasedNode, OUTPUT_DATASET_FIELDS_FIELD_NAME, Collections.emptyList());
 
+    ((ObjectNode) rebasedNode)
+        .set(
+            FINE_GRAINED_LINEAGES_FIELD_NAME,
+            FineGrainedLineageTemplateHelper.reconstructFineGrainedLineages(
+                rebasedNode.get(FINE_GRAINED_LINEAGES_FIELD_NAME)));
+
     return rebasedNode;
   }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/UpstreamLineageTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/UpstreamLineageTemplate.java
index e5da7551ac1a4..f28b35e3a5f4f 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/UpstreamLineageTemplate.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/UpstreamLineageTemplate.java
@@ -1,25 +1,17 @@
 package com.linkedin.metadata.aspect.patch.template.dataset;
 
-import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance;
 import static com.linkedin.metadata.Constants.*;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Streams;
-import com.linkedin.common.urn.Urn;
-import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.dataset.FineGrainedLineageArray;
 import com.linkedin.dataset.UpstreamArray;
 import com.linkedin.dataset.UpstreamLineage;
 import com.linkedin.metadata.aspect.patch.template.CompoundKeyTemplate;
+import com.linkedin.metadata.aspect.patch.template.FineGrainedLineageTemplateHelper;
 import java.util.Collections;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-import org.codehaus.plexus.util.StringUtils;
 
 public class UpstreamLineageTemplate extends CompoundKeyTemplate<UpstreamLineage> {
 
@@ -27,18 +19,6 @@ public class UpstreamLineageTemplate extends CompoundKeyTemplate<UpstreamLineage
   private static final String UPSTREAMS_FIELD_NAME = "upstreams";
   private static final String DATASET_FIELD_NAME = "dataset";
   private static final String FINE_GRAINED_LINEAGES_FIELD_NAME = "fineGrainedLineages";
-  private static final String FINE_GRAINED_UPSTREAM_TYPE = "upstreamType";
-  private static final String FINE_GRAINED_UPSTREAMS = "upstreams";
-  private static final String FINE_GRAINED_DOWNSTREAM_TYPE = "downstreamType";
-  private static final String FINE_GRAINED_DOWNSTREAMS = "downstreams";
-  private static final String FINE_GRAINED_TRANSFORMATION_OPERATION = "transformOperation";
-  private static final String FINE_GRAINED_CONFIDENCE_SCORE = "confidenceScore";
-  private static final String FINE_GRAINED_QUERY_ID = "query";
-
-  // Template support
-  private static final String NONE_TRANSFORMATION_TYPE = "NONE";
-  private static final Float DEFAULT_CONFIDENCE_SCORE = 1.0f;
-  private static final String DEFAULT_QUERY_ID = "NONE";
 
   @Override
   public UpstreamLineage getSubtype(RecordTemplate recordTemplate) throws ClassCastException {
@@ -72,7 +52,7 @@ public JsonNode transformFields(JsonNode baseNode) {
     ((ObjectNode) transformedNode)
         .set(
             FINE_GRAINED_LINEAGES_FIELD_NAME,
-            combineAndTransformFineGrainedLineages(
+            FineGrainedLineageTemplateHelper.combineAndTransformFineGrainedLineages(
                 transformedNode.get(FINE_GRAINED_LINEAGES_FIELD_NAME)));
 
     return transformedNode;
@@ -87,257 +67,8 @@ public JsonNode rebaseFields(JsonNode patched) {
     ((ObjectNode) rebasedNode)
         .set(
             FINE_GRAINED_LINEAGES_FIELD_NAME,
-            reconstructFineGrainedLineages(rebasedNode.get(FINE_GRAINED_LINEAGES_FIELD_NAME)));
+            FineGrainedLineageTemplateHelper.reconstructFineGrainedLineages(
+                rebasedNode.get(FINE_GRAINED_LINEAGES_FIELD_NAME)));
     return rebasedNode;
   }
-
-  /**
-   * Combines fine grained lineage array into a map using upstream and downstream types as keys,
-   * defaulting when not present. Due to this construction, patches will look like: path:
-   * /fineGrainedLineages/TRANSFORMATION_OPERATION/DOWNSTREAM_FIELD_URN/QUERY_ID/UPSTREAM_FIELD_URN,
-   * op: ADD/REMOVE, value: float (confidenceScore) Due to the way FineGrainedLineage was designed
-   * it doesn't necessarily have a consistent key we can reference, so this specialized method
-   * mimics the arrayFieldToMap of the super class with the specialization that it does not put the
-   * full value of the aspect at the end of the key, just the particular array. This prevents
-   * unintended overwrites through improper MCP construction that is technically allowed by the
-   * schema when combining under fields that form the natural key.
-   *
-   * @param fineGrainedLineages the fine grained lineage array node
-   * @return the modified {@link JsonNode} with array fields transformed to maps
-   */
-  private JsonNode combineAndTransformFineGrainedLineages(@Nullable JsonNode fineGrainedLineages) {
-    ObjectNode mapNode = instance.objectNode();
-    if (!(fineGrainedLineages instanceof ArrayNode) || fineGrainedLineages.isEmpty()) {
-      return mapNode;
-    }
-    JsonNode lineageCopy = fineGrainedLineages.deepCopy();
-
-    lineageCopy
-        .elements()
-        .forEachRemaining(
-            node -> {
-              JsonNode nodeClone = node.deepCopy();
-              String transformationOperation =
-                  nodeClone.has(FINE_GRAINED_TRANSFORMATION_OPERATION)
-                      ? nodeClone.get(FINE_GRAINED_TRANSFORMATION_OPERATION).asText()
-                      : NONE_TRANSFORMATION_TYPE;
-
-              if (!mapNode.has(transformationOperation)) {
-                mapNode.set(transformationOperation, instance.objectNode());
-              }
-              ObjectNode transformationOperationNode =
-                  (ObjectNode) mapNode.get(transformationOperation);
-
-              ArrayNode downstreams =
-                  nodeClone.has(FINE_GRAINED_DOWNSTREAMS)
-                      ? (ArrayNode) nodeClone.get(FINE_GRAINED_DOWNSTREAMS)
-                      : null;
-
-              if (downstreams == null || downstreams.size() != 1) {
-                throw new UnsupportedOperationException(
-                    "Patching not supported on fine grained lineages with not"
-                        + " exactly one downstream. Current fine grained lineage implementation is downstream derived and "
-                        + "patches are keyed on the root of this derivation.");
-              }
-
-              Float confidenceScore =
-                  nodeClone.has(FINE_GRAINED_CONFIDENCE_SCORE)
-                      ? nodeClone.get(FINE_GRAINED_CONFIDENCE_SCORE).floatValue()
-                      : DEFAULT_CONFIDENCE_SCORE;
-
-              String upstreamType =
-                  nodeClone.has(FINE_GRAINED_UPSTREAM_TYPE)
-                      ? nodeClone.get(FINE_GRAINED_UPSTREAM_TYPE).asText()
-                      : null;
-              String downstreamType =
-                  nodeClone.has(FINE_GRAINED_DOWNSTREAM_TYPE)
-                      ? nodeClone.get(FINE_GRAINED_DOWNSTREAM_TYPE).asText()
-                      : null;
-              ArrayNode upstreams =
-                  nodeClone.has(FINE_GRAINED_UPSTREAMS)
-                      ? (ArrayNode) nodeClone.get(FINE_GRAINED_UPSTREAMS)
-                      : null;
-
-              String queryId =
-                  nodeClone.has(FINE_GRAINED_QUERY_ID)
-                      ? nodeClone.get(FINE_GRAINED_QUERY_ID).asText()
-                      : DEFAULT_QUERY_ID;
-
-              if (upstreamType == null) {
-                // Determine default type
-                Urn upstreamUrn =
-                    upstreams != null ? UrnUtils.getUrn(upstreams.get(0).asText()) : null;
-                if (upstreamUrn != null
-                    && DATASET_ENTITY_NAME.equals(upstreamUrn.getEntityType())) {
-                  upstreamType = FINE_GRAINED_LINEAGE_DATASET_TYPE;
-                } else {
-                  upstreamType = FINE_GRAINED_LINEAGE_FIELD_SET_TYPE;
-                }
-              }
-
-              if (downstreamType == null) {
-                // Always use FIELD type, only support patches for single field downstream
-                downstreamType = FINE_GRAINED_LINEAGE_FIELD_TYPE;
-              }
-
-              String downstreamRoot = downstreams.get(0).asText();
-              if (!transformationOperationNode.has(downstreamRoot)) {
-                transformationOperationNode.set(downstreamRoot, instance.objectNode());
-              }
-              ObjectNode downstreamRootNode =
-                  (ObjectNode) transformationOperationNode.get(downstreamRoot);
-              if (!downstreamRootNode.has(queryId)) {
-                downstreamRootNode.set(queryId, instance.objectNode());
-              }
-              ObjectNode queryNode = (ObjectNode) downstreamRootNode.get(queryId);
-              if (upstreams != null) {
-                addUrnsToParent(
-                    queryNode, upstreams, confidenceScore, upstreamType, downstreamType);
-              }
-            });
-    return mapNode;
-  }
-
-  private void addUrnsToParent(
-      JsonNode parentNode,
-      ArrayNode urnsList,
-      Float confidenceScore,
-      String upstreamType,
-      String downstreamType) {
-    // Will overwrite repeat urns with different confidence scores with the most recently seen
-    ((ObjectNode) parentNode)
-        .setAll(
-            Streams.stream(urnsList.elements())
-                .map(JsonNode::asText)
-                .distinct()
-                .collect(
-                    Collectors.toMap(
-                        urn -> urn,
-                        urn ->
-                            mapToLineageValueNode(confidenceScore, upstreamType, downstreamType))));
-  }
-
-  private JsonNode mapToLineageValueNode(
-      Float confidenceScore, String upstreamType, String downstreamType) {
-    ObjectNode objectNode = instance.objectNode();
-    objectNode.set(FINE_GRAINED_CONFIDENCE_SCORE, instance.numberNode(confidenceScore));
-    objectNode.set(FINE_GRAINED_UPSTREAM_TYPE, instance.textNode(upstreamType));
-    objectNode.set(FINE_GRAINED_DOWNSTREAM_TYPE, instance.textNode(downstreamType));
-    return objectNode;
-  }
-
-  /**
-   * Takes the transformed fine grained lineages map from pre-processing and reconstructs an array
-   * of FineGrainedLineages Avoids producing side effects by copying nodes, use resulting node and
-   * not the original
-   *
-   * @param transformedFineGrainedLineages the transformed fine grained lineage map
-   * @return the modified {@link JsonNode} formatted consistent with the original schema
-   */
-  private ArrayNode reconstructFineGrainedLineages(JsonNode transformedFineGrainedLineages) {
-    if (transformedFineGrainedLineages instanceof ArrayNode) {
-      // We already have an ArrayNode, no need to transform. This happens during `replace`
-      // operations
-      return (ArrayNode) transformedFineGrainedLineages;
-    }
-    ObjectNode mapNode = (ObjectNode) transformedFineGrainedLineages;
-    ArrayNode fineGrainedLineages = instance.arrayNode();
-
-    mapNode
-        .fieldNames()
-        .forEachRemaining(
-            transformationOperation -> {
-              final ObjectNode transformationOperationNode =
-                  (ObjectNode) mapNode.get(transformationOperation);
-              transformationOperationNode
-                  .fieldNames()
-                  .forEachRemaining(
-                      downstreamName -> {
-                        final ObjectNode downstreamNode =
-                            (ObjectNode) transformationOperationNode.get(downstreamName);
-                        downstreamNode
-                            .fieldNames()
-                            .forEachRemaining(
-                                queryId ->
-                                    buildFineGrainedLineage(
-                                        downstreamName,
-                                        downstreamNode,
-                                        queryId,
-                                        transformationOperation,
-                                        fineGrainedLineages));
-                      });
-            });
-
-    return fineGrainedLineages;
-  }
-
-  private void buildFineGrainedLineage(
-      final String downstreamName,
-      final ObjectNode downstreamNode,
-      final String queryId,
-      final String transformationOperation,
-      final ArrayNode fineGrainedLineages) {
-    final ObjectNode fineGrainedLineage = instance.objectNode();
-    final ObjectNode queryNode = (ObjectNode) downstreamNode.get(queryId);
-    if (queryNode.isEmpty()) {
-      // Short circuit if no upstreams left
-      return;
-    }
-    ArrayNode downstream = instance.arrayNode();
-    downstream.add(instance.textNode(downstreamName));
-    // Set defaults, if found in sub nodes override, for confidenceScore take lowest
-    AtomicReference<Float> minimumConfidenceScore = new AtomicReference<>(DEFAULT_CONFIDENCE_SCORE);
-    AtomicReference<String> upstreamType =
-        new AtomicReference<>(FINE_GRAINED_LINEAGE_FIELD_SET_TYPE);
-    AtomicReference<String> downstreamType = new AtomicReference<>(FINE_GRAINED_LINEAGE_FIELD_TYPE);
-    ArrayNode upstreams = instance.arrayNode();
-    queryNode
-        .fieldNames()
-        .forEachRemaining(
-            upstream ->
-                processUpstream(
-                    queryNode,
-                    upstream,
-                    minimumConfidenceScore,
-                    upstreamType,
-                    downstreamType,
-                    upstreams));
-    fineGrainedLineage.set(FINE_GRAINED_DOWNSTREAMS, downstream);
-    fineGrainedLineage.set(FINE_GRAINED_UPSTREAMS, upstreams);
-    if (StringUtils.isNotBlank(queryId) && !DEFAULT_QUERY_ID.equals(queryId)) {
-      fineGrainedLineage.set(FINE_GRAINED_QUERY_ID, instance.textNode(queryId));
-    }
-    fineGrainedLineage.set(FINE_GRAINED_UPSTREAM_TYPE, instance.textNode(upstreamType.get()));
-    fineGrainedLineage.set(FINE_GRAINED_DOWNSTREAM_TYPE, instance.textNode(downstreamType.get()));
-    fineGrainedLineage.set(
-        FINE_GRAINED_CONFIDENCE_SCORE, instance.numberNode(minimumConfidenceScore.get()));
-    fineGrainedLineage.set(
-        FINE_GRAINED_TRANSFORMATION_OPERATION, instance.textNode(transformationOperation));
-    fineGrainedLineages.add(fineGrainedLineage);
-  }
-
-  private void processUpstream(
-      final ObjectNode queryNode,
-      final String upstream,
-      final AtomicReference<Float> minimumConfidenceScore,
-      final AtomicReference<String> upstreamType,
-      final AtomicReference<String> downstreamType,
-      final ArrayNode upstreams) {
-    final ObjectNode upstreamNode = (ObjectNode) queryNode.get(upstream);
-    if (upstreamNode.has(FINE_GRAINED_CONFIDENCE_SCORE)) {
-      Float scoreValue = upstreamNode.get(FINE_GRAINED_CONFIDENCE_SCORE).floatValue();
-      if (scoreValue <= minimumConfidenceScore.get()) {
-        minimumConfidenceScore.set(scoreValue);
-      }
-    }
-    // Set types to last encountered, should never change, but this at least tries to support
-    // other types being specified.
-    if (upstreamNode.has(FINE_GRAINED_UPSTREAM_TYPE)) {
-      upstreamType.set(upstreamNode.get(FINE_GRAINED_UPSTREAM_TYPE).asText());
-    }
-    if (upstreamNode.has(FINE_GRAINED_DOWNSTREAM_TYPE)) {
-      downstreamType.set(upstreamNode.get(FINE_GRAINED_DOWNSTREAM_TYPE).asText());
-    }
-    upstreams.add(instance.textNode(upstream));
-  }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java
index f99dd18d3c9c1..54ccd3877395f 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java
@@ -12,7 +12,7 @@
 @AllArgsConstructor
 @EqualsAndHashCode
 public abstract class PluginSpec {
-  protected static String ENTITY_WILDCARD = "*";
+  protected static String WILDCARD = "*";
 
   @Nonnull
   public abstract AspectPluginConfig getConfig();
@@ -50,7 +50,7 @@ protected boolean isEntityAspectSupported(
     return (getConfig().getSupportedEntityAspectNames().stream()
             .anyMatch(
                 supported ->
-                    ENTITY_WILDCARD.equals(supported.getEntityName())
+                    WILDCARD.equals(supported.getEntityName())
                         || supported.getEntityName().equals(entityName)))
         && isAspectSupported(aspectName);
   }
@@ -59,13 +59,16 @@ protected boolean isAspectSupported(@Nonnull String aspectName) {
     return getConfig().getSupportedEntityAspectNames().stream()
         .anyMatch(
             supported ->
-                ENTITY_WILDCARD.equals(supported.getAspectName())
+                WILDCARD.equals(supported.getAspectName())
                     || supported.getAspectName().equals(aspectName));
   }
 
   protected boolean isChangeTypeSupported(@Nullable ChangeType changeType) {
     return (changeType == null && getConfig().getSupportedOperations().isEmpty())
         || getConfig().getSupportedOperations().stream()
-            .anyMatch(supported -> supported.equalsIgnoreCase(String.valueOf(changeType)));
+            .anyMatch(
+                supported ->
+                    WILDCARD.equals(supported)
+                        || supported.equalsIgnoreCase(String.valueOf(changeType)));
   }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java
index dd8798ee89ae6..938cb2d5f99e6 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java
@@ -18,45 +18,39 @@ public static AspectValidationException forItem(BatchItem item, String msg) {
   }
 
   public static AspectValidationException forItem(BatchItem item, String msg, Exception e) {
-    return new AspectValidationException(
-        item.getChangeType(), item.getUrn(), item.getAspectName(), msg, SubType.VALIDATION, e);
+    return new AspectValidationException(item, msg, SubType.VALIDATION, e);
   }
 
   public static AspectValidationException forPrecondition(BatchItem item, String msg) {
     return forPrecondition(item, msg, null);
   }
 
+  public static AspectValidationException forFilter(BatchItem item, String msg) {
+    return new AspectValidationException(item, msg, SubType.FILTER);
+  }
+
   public static AspectValidationException forPrecondition(BatchItem item, String msg, Exception e) {
-    return new AspectValidationException(
-        item.getChangeType(), item.getUrn(), item.getAspectName(), msg, SubType.PRECONDITION, e);
+    return new AspectValidationException(item, msg, SubType.PRECONDITION, e);
   }
 
+  @Nonnull BatchItem item;
   @Nonnull ChangeType changeType;
   @Nonnull Urn entityUrn;
   @Nonnull String aspectName;
   @Nonnull SubType subType;
   @Nullable String msg;
 
-  public AspectValidationException(
-      @Nonnull ChangeType changeType,
-      @Nonnull Urn entityUrn,
-      @Nonnull String aspectName,
-      String msg,
-      SubType subType) {
-    this(changeType, entityUrn, aspectName, msg, subType, null);
+  public AspectValidationException(@Nonnull BatchItem item, String msg, SubType subType) {
+    this(item, msg, subType, null);
   }
 
   public AspectValidationException(
-      @Nonnull ChangeType changeType,
-      @Nonnull Urn entityUrn,
-      @Nonnull String aspectName,
-      @Nonnull String msg,
-      @Nullable SubType subType,
-      Exception e) {
+      @Nonnull BatchItem item, @Nonnull String msg, @Nullable SubType subType, Exception e) {
     super(msg, e);
-    this.changeType = changeType;
-    this.entityUrn = entityUrn;
-    this.aspectName = aspectName;
+    this.item = item;
+    this.changeType = item.getChangeType();
+    this.entityUrn = item.getUrn();
+    this.aspectName = item.getAspectName();
     this.msg = msg;
     this.subType = subType != null ? subType : SubType.VALIDATION;
   }
@@ -65,8 +59,12 @@ public Pair<Urn, String> getAspectGroup() {
     return Pair.of(entityUrn, aspectName);
   }
 
-  public static enum SubType {
+  public enum SubType {
+    // A validation exception is thrown
     VALIDATION,
-    PRECONDITION
+    // A failed precondition is thrown if the header constraints are not met
+    PRECONDITION,
+    // Exclude from processing further
+    FILTER
   }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java
index 007c196156b12..fc1fcb68029ce 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java
@@ -15,12 +15,30 @@
 public class ValidationExceptionCollection
     extends HashMap<Pair<Urn, String>, Set<AspectValidationException>> {
 
+  private final Set<Integer> failedHashCodes;
+  private final Set<Integer> filteredHashCodes;
+
+  public ValidationExceptionCollection() {
+    super();
+    this.failedHashCodes = new HashSet<>();
+    this.filteredHashCodes = new HashSet<>();
+  }
+
+  public boolean hasFatalExceptions() {
+    return !failedHashCodes.isEmpty();
+  }
+
   public static ValidationExceptionCollection newCollection() {
     return new ValidationExceptionCollection();
   }
 
   public void addException(AspectValidationException exception) {
     super.computeIfAbsent(exception.getAspectGroup(), key -> new HashSet<>()).add(exception);
+    if (!AspectValidationException.SubType.FILTER.equals(exception.getSubType())) {
+      failedHashCodes.add(exception.getItem().hashCode());
+    } else {
+      filteredHashCodes.add(exception.getItem().hashCode());
+    }
   }
 
   public void addException(BatchItem item, String message) {
@@ -28,8 +46,7 @@ public void addException(BatchItem item, String message) {
   }
 
   public void addException(BatchItem item, String message, Exception ex) {
-    super.computeIfAbsent(Pair.of(item.getUrn(), item.getAspectName()), key -> new HashSet<>())
-        .add(AspectValidationException.forItem(item, message, ex));
+    addException(AspectValidationException.forItem(item, message, ex));
   }
 
   public Stream<AspectValidationException> streamAllExceptions() {
@@ -41,7 +58,8 @@ public <T extends BatchItem> Collection<T> successful(Collection<T> items) {
   }
 
   public <T extends BatchItem> Stream<T> streamSuccessful(Stream<T> items) {
-    return items.filter(i -> !this.containsKey(Pair.of(i.getUrn(), i.getAspectName())));
+    return items.filter(
+        i -> !failedHashCodes.contains(i.hashCode()) && !filteredHashCodes.contains(i.hashCode()));
   }
 
   public <T extends BatchItem> Collection<T> exceptions(Collection<T> items) {
@@ -49,7 +67,7 @@ public <T extends BatchItem> Collection<T> exceptions(Collection<T> items) {
   }
 
   public <T extends BatchItem> Stream<T> streamExceptions(Stream<T> items) {
-    return items.filter(i -> this.containsKey(Pair.of(i.getUrn(), i.getAspectName())));
+    return items.filter(i -> failedHashCodes.contains(i.hashCode()));
   }
 
   @Override
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/CreateIfNotExistsValidator.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/CreateIfNotExistsValidator.java
index 2ad885dc9fdd2..9b9d8f49d8462 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/CreateIfNotExistsValidator.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/CreateIfNotExistsValidator.java
@@ -25,6 +25,8 @@
 @Getter
 @Accessors(chain = true)
 public class CreateIfNotExistsValidator extends AspectPayloadValidator {
+  public static final String FILTER_EXCEPTION_HEADER = "If-None-Match";
+  public static final String FILTER_EXCEPTION_VALUE = "*";
 
   @Nonnull private AspectPluginConfig config;
 
@@ -49,11 +51,17 @@ protected Stream<AspectValidationException> validatePreCommitAspects(
             .filter(item -> ChangeType.CREATE_ENTITY.equals(item.getChangeType()))
             .collect(Collectors.toSet())) {
       // if the key aspect is missing in the batch, the entity exists and CREATE_ENTITY should be
-      // denied
+      // denied or dropped
       if (!entityKeyMap.containsKey(createEntityItem.getUrn())) {
-        exceptions.addException(
-            createEntityItem,
-            "Cannot perform CREATE_ENTITY if not exists since the entity key already exists.");
+        if (isPrecondition(createEntityItem)) {
+          exceptions.addException(
+              AspectValidationException.forFilter(
+                  createEntityItem, "Dropping write per precondition header If-None-Match: *"));
+        } else {
+          exceptions.addException(
+              createEntityItem,
+              "Cannot perform CREATE_ENTITY if not exists since the entity key already exists.");
+        }
       }
     }
 
@@ -61,10 +69,16 @@ protected Stream<AspectValidationException> validatePreCommitAspects(
         changeMCPs.stream()
             .filter(item -> ChangeType.CREATE.equals(item.getChangeType()))
             .collect(Collectors.toSet())) {
-      // if a CREATE item has a previous value, should be denied
+      // if a CREATE item has a previous value, should be denied or dropped
       if (createItem.getPreviousRecordTemplate() != null) {
-        exceptions.addException(
-            createItem, "Cannot perform CREATE since the aspect already exists.");
+        if (isPrecondition(createItem)) {
+          exceptions.addException(
+              AspectValidationException.forFilter(
+                  createItem, "Dropping write per precondition header If-None-Match: *"));
+        } else {
+          exceptions.addException(
+              createItem, "Cannot perform CREATE since the aspect already exists.");
+        }
       }
     }
 
@@ -77,4 +91,10 @@ protected Stream<AspectValidationException> validateProposedAspects(
       @Nonnull RetrieverContext retrieverContext) {
     return Stream.empty();
   }
+
+  private static boolean isPrecondition(ChangeMCP item) {
+    return item.getHeader(FILTER_EXCEPTION_HEADER)
+        .map(FILTER_EXCEPTION_VALUE::equals)
+        .orElse(false);
+  }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/FieldPathValidator.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/FieldPathValidator.java
new file mode 100644
index 0000000000000..7c279254e1bc3
--- /dev/null
+++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/FieldPathValidator.java
@@ -0,0 +1,116 @@
+package com.linkedin.metadata.aspect.validation;
+
+import static com.linkedin.metadata.Constants.*;
+
+import com.linkedin.metadata.aspect.RetrieverContext;
+import com.linkedin.metadata.aspect.batch.BatchItem;
+import com.linkedin.metadata.aspect.batch.ChangeMCP;
+import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
+import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
+import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection;
+import com.linkedin.schema.EditableSchemaFieldInfo;
+import com.linkedin.schema.EditableSchemaMetadata;
+import com.linkedin.schema.SchemaField;
+import com.linkedin.schema.SchemaMetadata;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+/**
+ * 1. Validates the Schema Field Path specification, specifically that all field IDs must be unique
+ * across all fields within a schema. 2. Validates that the field path id is not empty.
+ *
+ * @see <a href="https://datahubproject.io/docs/advanced/field-path-spec-v2/#requirements">Field
+ *     Path V2 docs</a>
+ */
+@Setter
+@Getter
+@Accessors(chain = true)
+public class FieldPathValidator extends AspectPayloadValidator {
+  @Nonnull private AspectPluginConfig config;
+
+  /** Prevent any MCP for SchemaMetadata where field ids are duplicated. */
+  @Override
+  protected Stream<AspectValidationException> validateProposedAspects(
+      @Nonnull Collection<? extends BatchItem> mcpItems,
+      @Nonnull RetrieverContext retrieverContext) {
+
+    ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection();
+
+    mcpItems.forEach(
+        i -> {
+          if (i.getAspectName().equals(SCHEMA_METADATA_ASPECT_NAME)) {
+            processSchemaMetadataAspect(i, exceptions);
+          } else {
+            processEditableSchemaMetadataAspect(i, exceptions);
+          }
+        });
+
+    return exceptions.streamAllExceptions();
+  }
+
+  @Override
+  protected Stream<AspectValidationException> validatePreCommitAspects(
+      @Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull RetrieverContext retrieverContext) {
+    return Stream.of();
+  }
+
+  private static void processEditableSchemaMetadataAspect(
+      BatchItem i, ValidationExceptionCollection exceptions) {
+    final EditableSchemaMetadata schemaMetadata = i.getAspect(EditableSchemaMetadata.class);
+    final long uniquePaths =
+        validateAndCount(
+            i,
+            schemaMetadata.getEditableSchemaFieldInfo().stream()
+                .map(EditableSchemaFieldInfo::getFieldPath),
+            exceptions);
+
+    if (uniquePaths != schemaMetadata.getEditableSchemaFieldInfo().size()) {
+      exceptions.addException(
+          i,
+          String.format(
+              "Cannot perform %s action on proposal. EditableSchemaMetadata aspect has duplicated field paths",
+              i.getChangeType()));
+    }
+  }
+
+  private static void processSchemaMetadataAspect(
+      BatchItem i, ValidationExceptionCollection exceptions) {
+    final SchemaMetadata schemaMetadata = i.getAspect(SchemaMetadata.class);
+    final long uniquePaths =
+        validateAndCount(
+            i, schemaMetadata.getFields().stream().map(SchemaField::getFieldPath), exceptions);
+
+    if (uniquePaths != schemaMetadata.getFields().size()) {
+      exceptions.addException(
+          i,
+          String.format(
+              "Cannot perform %s action on proposal. SchemaMetadata aspect has duplicated field paths",
+              i.getChangeType()));
+    }
+  }
+
+  private static long validateAndCount(
+      BatchItem i, Stream<String> fieldPaths, ValidationExceptionCollection exceptions) {
+    return fieldPaths
+        .distinct()
+        // inspect the stream of fieldPath validation errors since we're already iterating
+        .peek(
+            fieldPath ->
+                validateFieldPath(fieldPath)
+                    .ifPresent(message -> exceptions.addException(i, message)))
+        .count();
+  }
+
+  private static Optional<String> validateFieldPath(String fieldPath) {
+    if (fieldPath == null || fieldPath.isEmpty()) {
+      return Optional.of("SchemaMetadata aspect has empty field path.");
+    }
+    return Optional.empty();
+  }
+}
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/entity/GenericScrollIterator.java b/entity-registry/src/main/java/com/linkedin/metadata/entity/GenericScrollIterator.java
new file mode 100644
index 0000000000000..3f393e0b894aa
--- /dev/null
+++ b/entity-registry/src/main/java/com/linkedin/metadata/entity/GenericScrollIterator.java
@@ -0,0 +1,35 @@
+package com.linkedin.metadata.entity;
+
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.search.ScrollResult;
+import java.util.Iterator;
+import java.util.List;
+import javax.annotation.Nonnull;
+import lombok.Builder;
+
+/**
+ * Fetches pages of structured properties which have been applied to an entity urn with a specified
+ * filter
+ */
+@Builder
+public class GenericScrollIterator implements Iterator<ScrollResult> {
+  @Nonnull private final Filter filter;
+  @Nonnull private final List<String> entities;
+  @Nonnull private final SearchRetriever searchRetriever;
+  private int count;
+  @Builder.Default private String scrollId = null;
+  @Builder.Default private boolean started = false;
+
+  @Override
+  public boolean hasNext() {
+    return !started || scrollId != null;
+  }
+
+  @Override
+  public ScrollResult next() {
+    started = true;
+    ScrollResult result = searchRetriever.scroll(entities, filter, scrollId, count);
+    scrollId = result.getScrollId();
+    return result;
+  }
+}
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/entity/SearchRetriever.java b/entity-registry/src/main/java/com/linkedin/metadata/entity/SearchRetriever.java
index eaa106b8d1f63..d4894c97015f8 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/entity/SearchRetriever.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/entity/SearchRetriever.java
@@ -2,6 +2,7 @@
 
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.search.ScrollResult;
+import com.linkedin.metadata.search.SearchEntityArray;
 import java.util.List;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -21,4 +22,22 @@ ScrollResult scroll(
       @Nullable Filter filters,
       @Nullable String scrollId,
       int count);
+
+  SearchRetriever EMPTY = new EmptySearchRetriever();
+
+  class EmptySearchRetriever implements SearchRetriever {
+
+    @Override
+    public ScrollResult scroll(
+        @Nonnull List<String> entities,
+        @Nullable Filter filters,
+        @Nullable String scrollId,
+        int count) {
+      ScrollResult empty = new ScrollResult();
+      empty.setEntities(new SearchEntityArray());
+      empty.setNumEntities(0);
+      empty.setPageSize(0);
+      return empty;
+    }
+  }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java
index 885be5e00b945..612f923d5d68b 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java
@@ -6,7 +6,6 @@
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD;
-import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD_PREFIX;
 
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.Status;
@@ -21,6 +20,7 @@
 import com.linkedin.structured.PrimitivePropertyValue;
 import com.linkedin.structured.StructuredProperties;
 import com.linkedin.structured.StructuredPropertyDefinition;
+import com.linkedin.structured.StructuredPropertySettings;
 import com.linkedin.structured.StructuredPropertyValueAssignment;
 import com.linkedin.structured.StructuredPropertyValueAssignmentArray;
 import com.linkedin.util.Pair;
@@ -33,6 +33,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -46,6 +47,11 @@ private StructuredPropertyUtils() {}
   static final Date MIN_DATE = Date.valueOf("1000-01-01");
   static final Date MAX_DATE = Date.valueOf("9999-12-31");
 
+  public static final String INVALID_SETTINGS_MESSAGE =
+      "Cannot have property isHidden = true while other display location settings are also true.";
+  public static final String ONLY_ONE_BADGE =
+      "Cannot have more than one property set with show as badge. Property urns currently set: ";
+
   public static LogicalValueType getLogicalValueType(
       StructuredPropertyDefinition structuredPropertyDefinition) {
     return getLogicalValueType(structuredPropertyDefinition.getValueType());
@@ -120,12 +126,13 @@ public static Optional<String> toStructuredPropertyFacetName(
       lookupDefinitionFromFilterOrFacetName(
           @Nonnull String fieldOrFacetName, @Nullable AspectRetriever aspectRetriever) {
     if (fieldOrFacetName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD + ".")) {
-      String fqn =
-          fieldOrFacetName
-              .substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1)
-              .replace(".keyword", "")
-              .replace(".delimited", "");
+      // Coming in from the UI this is structuredProperties.<FQN> + any particular specifier for
+      // subfield (.keyword etc)
+      String fqn = fieldOrFacetToFQN(fieldOrFacetName);
+
+      // FQN Maps directly to URN with urn:li:structuredProperties:FQN
       Urn urn = toURNFromFQN(fqn);
+
       Map<Urn, Map<String, Aspect>> result =
           Objects.requireNonNull(aspectRetriever)
               .getLatestAspectObjects(
@@ -178,7 +185,7 @@ public static String toElasticsearchFieldName(
   /**
    * Return an elasticsearch type from structured property type
    *
-   * @param fieldName filter or facet field name
+   * @param fieldName filter or facet field name - must match actual FQN of structured prop
    * @param aspectRetriever aspect retriever
    * @return elasticsearch type
    */
@@ -223,9 +230,8 @@ public static void validateStructuredPropertyFQN(
    * @param fqn structured property's fqn
    * @return the expected structured property urn
    */
-  private static Urn toURNFromFQN(@Nonnull String fqn) {
-    return UrnUtils.getUrn(
-        String.join(":", "urn:li", STRUCTURED_PROPERTY_ENTITY_NAME, fqn.replace('_', '.')));
+  public static Urn toURNFromFQN(@Nonnull String fqn) {
+    return UrnUtils.getUrn(String.join(":", "urn:li", STRUCTURED_PROPERTY_ENTITY_NAME, fqn));
   }
 
   public static void validateFilter(
@@ -235,12 +241,13 @@ public static void validateFilter(
       return;
     }
 
-    Set<String> fieldNames = new HashSet<>();
+    Set<String> fqns = new HashSet<>();
 
     if (filter.getCriteria() != null) {
       for (Criterion c : filter.getCriteria()) {
         if (c.getField().startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) {
-          fieldNames.add(stripStructuredPropertyPrefix(c.getField()));
+          String fqn = fieldOrFacetToFQN(c.getField());
+          fqns.add(fqn);
         }
       }
     }
@@ -249,24 +256,23 @@ public static void validateFilter(
       for (ConjunctiveCriterion cc : filter.getOr()) {
         for (Criterion c : cc.getAnd()) {
           if (c.getField().startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) {
-            fieldNames.add(stripStructuredPropertyPrefix(c.getField()));
+            String fqn = fieldOrFacetToFQN(c.getField());
+            fqns.add(fqn);
           }
         }
       }
     }
 
-    if (!fieldNames.isEmpty()) {
-      validateStructuredPropertyFQN(fieldNames, Objects.requireNonNull(aspectRetriever));
+    if (!fqns.isEmpty()) {
+      validateStructuredPropertyFQN(fqns, Objects.requireNonNull(aspectRetriever));
     }
   }
 
-  private static String stripStructuredPropertyPrefix(String s) {
-    if (s.startsWith(STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD_PREFIX)) {
-      return s.substring(STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD.length() + 1).split("[.]")[0];
-    } else if (s.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) {
-      return s.substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1).split("[.]")[0];
-    }
-    return s;
+  private static String fieldOrFacetToFQN(String fieldOrFacet) {
+    return fieldOrFacet
+        .substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1)
+        .replace(".keyword", "")
+        .replace(".delimited", "");
   }
 
   public static Date toDate(PrimitivePropertyValue value) throws DateTimeParseException {
@@ -356,4 +362,48 @@ private static Pair<StructuredPropertyValueAssignmentArray, Boolean> filterValue
           true);
     }
   }
+
+  /*
+   * We accept both ID and qualifiedName as inputs when creating a structured property. However,
+   * these two fields should ALWAYS be the same. If they don't provide either, use a UUID for both.
+   * If they provide both, ensure they are the same otherwise throw. Otherwise, use what is provided.
+   */
+  public static String getPropertyId(
+      @Nullable final String inputId, @Nullable final String inputQualifiedName) {
+    if (inputId != null && inputQualifiedName != null && !inputId.equals(inputQualifiedName)) {
+      throw new IllegalArgumentException(
+          "Qualified name and the ID of a structured property must match");
+    }
+
+    String id = UUID.randomUUID().toString();
+
+    if (inputQualifiedName != null) {
+      id = inputQualifiedName;
+    } else if (inputId != null) {
+      id = inputId;
+    }
+
+    return id;
+  }
+
+  /*
+   * Ensure that a structured property settings aspect is valid by ensuring that if isHidden is true,
+   * the other fields concerning display locations are false;
+   */
+  public static boolean validatePropertySettings(
+      StructuredPropertySettings settings, boolean shouldThrow) {
+    if (settings.isIsHidden()) {
+      if (settings.isShowInSearchFilters()
+          || settings.isShowInAssetSummary()
+          || settings.isShowAsAssetBadge()
+          || settings.isShowInColumnsTable()) {
+        if (shouldThrow) {
+          throw new IllegalArgumentException(INVALID_SETTINGS_MESSAGE);
+        } else {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java
index bef7782d8f7c9..f4dc2ec2f0cd5 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java
@@ -14,7 +14,7 @@
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
-import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 /** Extracts fields from a RecordTemplate based on the appropriate {@link FieldSpec}. */
 public class FieldExtractor {
@@ -30,15 +30,34 @@ private static long getNumArrayWildcards(PathSpec pathSpec) {
 
   // Extract the value of each field in the field specs from the input record
   public static <T extends FieldSpec> Map<T, List<Object>> extractFields(
-      @Nonnull RecordTemplate record, List<T> fieldSpecs) {
-    return extractFields(record, fieldSpecs, MAX_VALUE_LENGTH);
+      @Nullable RecordTemplate record, List<T> fieldSpecs) {
+    return extractFields(record, fieldSpecs, false);
   }
 
   public static <T extends FieldSpec> Map<T, List<Object>> extractFields(
-      @Nonnull RecordTemplate record, List<T> fieldSpecs, int maxValueLength) {
+      @Nullable RecordTemplate record, List<T> fieldSpecs, boolean requiredFieldExtract) {
+    return extractFields(record, fieldSpecs, MAX_VALUE_LENGTH, requiredFieldExtract);
+  }
+
+  public static <T extends FieldSpec> Map<T, List<Object>> extractFields(
+      @Nullable RecordTemplate record, List<T> fieldSpecs, int maxValueLength) {
+    return extractFields(record, fieldSpecs, maxValueLength, false);
+  }
+
+  public static <T extends FieldSpec> Map<T, List<Object>> extractFields(
+      @Nullable RecordTemplate record,
+      List<T> fieldSpecs,
+      int maxValueLength,
+      boolean requiredFieldExtract) {
     final Map<T, List<Object>> extractedFields = new HashMap<>();
     for (T fieldSpec : fieldSpecs) {
-      Optional<Object> value = RecordUtils.getFieldValue(record, fieldSpec.getPath());
+      if (requiredFieldExtract && record == null) {
+        throw new IllegalArgumentException(
+            "Field extraction is required and the RecordTemplate is null");
+      }
+      Optional<Object> value =
+          Optional.ofNullable(record)
+              .flatMap(maybeRecord -> RecordUtils.getFieldValue(maybeRecord, fieldSpec.getPath()));
       if (!value.isPresent()) {
         extractedFields.put(fieldSpec, Collections.emptyList());
       } else {
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoader.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoader.java
index 4f2e5a106ae79..531537852109b 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoader.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoader.java
@@ -6,7 +6,6 @@
 import com.linkedin.metadata.models.registry.config.LoadStatus;
 import com.linkedin.util.Pair;
 import java.io.File;
-import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.nio.file.Files;
@@ -204,8 +203,8 @@ private void loadOneRegistry(
       loadResultBuilder.plugins(entityRegistry.getPluginFactory().getPluginLoadResult());
 
       log.info("Loaded registry {} successfully", entityRegistry);
-    } catch (RuntimeException | EntityRegistryException | IOException e) {
-      log.debug("{}: Failed to load registry {} with {}", this, registryName, e.getMessage());
+    } catch (Exception | EntityRegistryException e) {
+      log.error("{}: Failed to load registry {} with {}", this, registryName, e.getMessage(), e);
       StringWriter sw = new StringWriter();
       PrintWriter pw = new PrintWriter(sw);
       e.printStackTrace(pw);
diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilderTest.java
new file mode 100644
index 0000000000000..612282b7c0238
--- /dev/null
+++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilderTest.java
@@ -0,0 +1,280 @@
+package com.linkedin.metadata.aspect.patch.builder;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.linkedin.common.TimeStamp;
+import com.linkedin.common.urn.Urn;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.lang3.tuple.ImmutableTriple;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class DataFlowInfoPatchBuilderTest {
+
+  private TestableDataFlowInfoPatchBuilder builder;
+  private static final String TEST_URN = "urn:li:dataFlow:(test,flow1,PROD)";
+
+  // Test helper class to expose protected method
+  private static class TestableDataFlowInfoPatchBuilder extends DataFlowInfoPatchBuilder {
+    public List<ImmutableTriple<String, String, JsonNode>> getTestPathValues() {
+      return getPathValues();
+    }
+  }
+
+  @BeforeMethod
+  public void setup() throws URISyntaxException {
+    builder = new TestableDataFlowInfoPatchBuilder();
+    builder.urn(Urn.createFromString(TEST_URN));
+  }
+
+  @Test
+  public void testBuildDoesNotAffectPathValues() throws URISyntaxException {
+    String testName = "testFlow";
+    String testDescription = "Test description";
+
+    builder.setName(testName).setDescription(testDescription).addCustomProperty("key1", "value1");
+
+    // First call build()
+    builder.build();
+
+    // Then verify we can still access pathValues and they're correct
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 3);
+
+    // Verify the operations are still intact
+    assertEquals(pathValues.get(0).getLeft(), "add");
+    assertEquals(pathValues.get(0).getMiddle(), "/name");
+    assertEquals(pathValues.get(0).getRight().asText(), testName);
+
+    assertEquals(pathValues.get(1).getLeft(), "add");
+    assertEquals(pathValues.get(1).getMiddle(), "/description");
+    assertEquals(pathValues.get(1).getRight().asText(), testDescription);
+
+    assertEquals(pathValues.get(2).getLeft(), "add");
+    assertTrue(pathValues.get(2).getMiddle().startsWith("/customProperties/"));
+    assertEquals(pathValues.get(2).getRight().asText(), "value1");
+
+    // Verify we can call build() again without issues
+    builder.build();
+
+    // And verify pathValues are still accessible and correct
+    pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 3);
+  }
+
+  @Test
+  public void testSetName() {
+    String testName = "testFlow";
+    builder.setName(testName);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertEquals(operation.getMiddle(), "/name");
+    assertEquals(operation.getRight().asText(), testName);
+  }
+
+  @Test
+  public void testSetDescription() {
+    String testDescription = "Test description";
+    builder.setDescription(testDescription);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertEquals(operation.getMiddle(), "/description");
+    assertEquals(operation.getRight().asText(), testDescription);
+  }
+
+  @Test
+  public void testSetDescriptionNull() {
+    builder.setDescription(null);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "remove");
+    assertEquals(operation.getMiddle(), "/description");
+    assertNull(operation.getRight());
+  }
+
+  @Test
+  public void testSetProject() {
+    String testProject = "testProject";
+    builder.setProject(testProject);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertEquals(operation.getMiddle(), "/project");
+    assertEquals(operation.getRight().asText(), testProject);
+  }
+
+  @Test
+  public void testSetProjectNull() {
+    builder.setProject(null);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "remove");
+    assertEquals(operation.getMiddle(), "/project");
+    assertNull(operation.getRight());
+  }
+
+  @Test
+  public void testSetCreated() throws URISyntaxException {
+    long time = System.currentTimeMillis();
+    String actor = "urn:li:corpuser:testUser";
+    TimeStamp created = new TimeStamp();
+    created.setTime(time);
+    created.setActor(Urn.createFromString(actor));
+
+    builder.setCreated(created);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertEquals(operation.getMiddle(), "/created");
+    JsonNode createdNode = operation.getRight();
+    assertTrue(createdNode.isObject());
+    assertEquals(createdNode.get("time").asLong(), time);
+    assertEquals(createdNode.get("actor").asText(), actor);
+  }
+
+  @Test
+  public void testSetCreatedNull() {
+    builder.setCreated(null);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "remove");
+    assertEquals(operation.getMiddle(), "/created");
+    assertNull(operation.getRight());
+  }
+
+  @Test
+  public void testSetLastModified() throws URISyntaxException {
+    long time = System.currentTimeMillis();
+    String actor = "urn:li:corpuser:testUser";
+    TimeStamp lastModified = new TimeStamp();
+    lastModified.setTime(time);
+    lastModified.setActor(Urn.createFromString(actor));
+
+    builder.setLastModified(lastModified);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertEquals(operation.getMiddle(), "/lastModified");
+    JsonNode lastModifiedNode = operation.getRight();
+    assertTrue(lastModifiedNode.isObject());
+    assertEquals(lastModifiedNode.get("time").asLong(), time);
+    assertEquals(lastModifiedNode.get("actor").asText(), actor);
+  }
+
+  @Test
+  public void testSetLastModifiedNull() {
+    builder.setLastModified(null);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "remove");
+    assertEquals(operation.getMiddle(), "/lastModified");
+    assertNull(operation.getRight());
+  }
+
+  @Test
+  public void testAddCustomProperties() {
+    builder.addCustomProperty("key1", "value1").addCustomProperty("key2", "value2");
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 2);
+
+    pathValues.forEach(
+        operation -> {
+          assertEquals(operation.getLeft(), "add");
+          assertTrue(operation.getMiddle().startsWith("/customProperties/"));
+          assertTrue(operation.getRight().isTextual());
+        });
+  }
+
+  @Test
+  public void testRemoveCustomProperty() {
+    builder.removeCustomProperty("key1");
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "remove");
+    assertEquals(operation.getMiddle(), "/customProperties/key1");
+    assertNull(operation.getRight());
+  }
+
+  @Test
+  public void testSetCustomProperties() {
+    Map<String, String> properties = new HashMap<>();
+    properties.put("key1", "value1");
+    properties.put("key2", "value2");
+
+    builder.setCustomProperties(properties);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertEquals(operation.getMiddle(), "/customProperties");
+    assertTrue(operation.getRight().isObject());
+  }
+}
diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilderTest.java
new file mode 100644
index 0000000000000..dc141863e2443
--- /dev/null
+++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilderTest.java
@@ -0,0 +1,237 @@
+package com.linkedin.metadata.aspect.patch.builder;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.linkedin.common.Edge;
+import com.linkedin.common.urn.DataJobUrn;
+import com.linkedin.common.urn.DatasetUrn;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.metadata.graph.LineageDirection;
+import java.net.URISyntaxException;
+import java.util.List;
+import org.apache.commons.lang3.tuple.ImmutableTriple;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class DataJobInputOutputPatchBuilderTest {
+
+  private TestableDataJobInputOutputPatchBuilder builder;
+  private static final String TEST_DATAJOB_URN =
+      "urn:li:dataJob:(urn:li:dataFlow:(test,flow1,PROD),job1)";
+  private static final String TEST_DATASET_URN =
+      "urn:li:dataset:(urn:li:dataPlatform:hive,SampleTable,PROD)";
+  private static final String TEST_DATASET_FIELD_URN =
+      "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleTable,PROD),id)";
+
+  // Test helper class to expose protected method
+  private static class TestableDataJobInputOutputPatchBuilder
+      extends DataJobInputOutputPatchBuilder {
+    public List<ImmutableTriple<String, String, JsonNode>> getTestPathValues() {
+      return getPathValues();
+    }
+  }
+
+  @BeforeMethod
+  public void setup() throws URISyntaxException {
+    builder = new TestableDataJobInputOutputPatchBuilder();
+    builder.urn(Urn.createFromString(TEST_DATAJOB_URN));
+  }
+
+  @Test
+  public void testBuildDoesNotAffectPathValues() throws URISyntaxException {
+    DataJobUrn dataJobUrn = DataJobUrn.createFromString(TEST_DATAJOB_URN);
+    DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN);
+
+    builder
+        .addInputDatajobEdge(dataJobUrn)
+        .addInputDatasetEdge(datasetUrn)
+        .addOutputDatasetEdge(datasetUrn);
+
+    // First call build()
+    builder.build();
+
+    // Then verify we can still access pathValues and they're correct
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 3);
+
+    // Verify we can call build() again without issues
+    builder.build();
+
+    // And verify pathValues are still accessible and correct
+    pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 3);
+  }
+
+  @Test
+  public void testAddInputDatajobEdge() throws URISyntaxException {
+    DataJobUrn dataJobUrn = DataJobUrn.createFromString(TEST_DATAJOB_URN);
+    builder.addInputDatajobEdge(dataJobUrn);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertTrue(operation.getMiddle().startsWith("/inputDatajobEdges/"));
+    assertTrue(operation.getRight().isObject());
+    assertEquals(operation.getRight().get("destinationUrn").asText(), dataJobUrn.toString());
+  }
+
+  @Test
+  public void testRemoveInputDatajobEdge() throws URISyntaxException {
+    DataJobUrn dataJobUrn = DataJobUrn.createFromString(TEST_DATAJOB_URN);
+    builder.removeInputDatajobEdge(dataJobUrn);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "remove");
+    assertTrue(operation.getMiddle().startsWith("/inputDatajobEdges/"));
+    assertNull(operation.getRight());
+  }
+
+  @Test
+  public void testAddInputDatasetEdge() throws URISyntaxException {
+    DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN);
+    builder.addInputDatasetEdge(datasetUrn);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertTrue(operation.getMiddle().startsWith("/inputDatasetEdges/"));
+    assertTrue(operation.getRight().isObject());
+    assertEquals(operation.getRight().get("destinationUrn").asText(), datasetUrn.toString());
+  }
+
+  @Test
+  public void testRemoveInputDatasetEdge() throws URISyntaxException {
+    DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN);
+    builder.removeInputDatasetEdge(datasetUrn);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "remove");
+    assertTrue(operation.getMiddle().startsWith("/inputDatasetEdges/"));
+    assertNull(operation.getRight());
+  }
+
+  @Test
+  public void testAddOutputDatasetEdge() throws URISyntaxException {
+    DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN);
+    builder.addOutputDatasetEdge(datasetUrn);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertTrue(operation.getMiddle().startsWith("/outputDatasetEdges/"));
+    assertTrue(operation.getRight().isObject());
+    assertEquals(operation.getRight().get("destinationUrn").asText(), datasetUrn.toString());
+  }
+
+  @Test
+  public void testAddInputDatasetField() throws URISyntaxException {
+    Urn fieldUrn = Urn.createFromString(TEST_DATASET_FIELD_URN);
+    builder.addInputDatasetField(fieldUrn);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertTrue(operation.getMiddle().startsWith("/inputDatasetFields/"));
+    assertTrue(operation.getRight().isTextual());
+    assertEquals(operation.getRight().asText(), fieldUrn.toString());
+  }
+
+  @Test
+  public void testRemoveInputDatasetField() throws URISyntaxException {
+    Urn fieldUrn = Urn.createFromString(TEST_DATASET_FIELD_URN);
+    builder.removeInputDatasetField(fieldUrn);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "remove");
+    assertTrue(operation.getMiddle().startsWith("/inputDatasetFields/"));
+    assertNull(operation.getRight());
+  }
+
+  @Test
+  public void testAddOutputDatasetField() throws URISyntaxException {
+    Urn fieldUrn = Urn.createFromString(TEST_DATASET_FIELD_URN);
+    builder.addOutputDatasetField(fieldUrn);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertTrue(operation.getMiddle().startsWith("/outputDatasetFields/"));
+    assertTrue(operation.getRight().isTextual());
+    assertEquals(operation.getRight().asText(), fieldUrn.toString());
+  }
+
+  @Test
+  public void testAddEdgeWithDirection() throws URISyntaxException {
+    DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN);
+    Edge edge = new Edge();
+    edge.setDestinationUrn(datasetUrn);
+
+    builder.addEdge(edge, LineageDirection.UPSTREAM);
+    builder.build();
+
+    List<ImmutableTriple<String, String, JsonNode>> pathValues = builder.getTestPathValues();
+    assertNotNull(pathValues);
+    assertEquals(pathValues.size(), 1);
+
+    ImmutableTriple<String, String, JsonNode> operation = pathValues.get(0);
+    assertEquals(operation.getLeft(), "add");
+    assertTrue(operation.getMiddle().startsWith("/inputDatasetEdges/"));
+    assertTrue(operation.getRight().isObject());
+    assertEquals(operation.getRight().get("destinationUrn").asText(), datasetUrn.toString());
+  }
+
+  @Test
+  public void testInvalidEntityTypeThrowsException() throws URISyntaxException {
+    Urn invalidUrn = Urn.createFromString("urn:li:glossaryTerm:invalid");
+    Edge edge = new Edge();
+    edge.setDestinationUrn(invalidUrn);
+
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> {
+          builder.addEdge(edge, LineageDirection.UPSTREAM);
+        });
+  }
+}
diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DataJobInputOutputTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DataJobInputOutputTemplateTest.java
new file mode 100644
index 0000000000000..d2a26221a3bb9
--- /dev/null
+++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DataJobInputOutputTemplateTest.java
@@ -0,0 +1,255 @@
+package com.linkedin.metadata.aspect.patch.template;
+
+import static com.linkedin.metadata.utils.GenericRecordUtils.*;
+import static org.testng.Assert.*;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.linkedin.common.UrnArray;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.DataMap;
+import com.linkedin.datajob.DataJobInputOutput;
+import com.linkedin.dataset.FineGrainedLineage;
+import com.linkedin.dataset.FineGrainedLineageDownstreamType;
+import com.linkedin.dataset.FineGrainedLineageUpstreamType;
+import com.linkedin.metadata.aspect.patch.template.datajob.DataJobInputOutputTemplate;
+import jakarta.json.Json;
+import jakarta.json.JsonObjectBuilder;
+import jakarta.json.JsonPatch;
+import jakarta.json.JsonPatchBuilder;
+import jakarta.json.JsonValue;
+import org.testng.annotations.Test;
+
+public class DataJobInputOutputTemplateTest {
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+  @Test
+  public void testPatchUpstream() throws Exception {
+    DataJobInputOutputTemplate dataJobInputOutputTemplate = new DataJobInputOutputTemplate();
+    DataJobInputOutput dataJobInputOutput = dataJobInputOutputTemplate.getDefault();
+    JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder();
+
+    JsonObjectBuilder fineGrainedLineageNode = Json.createObjectBuilder();
+    JsonValue upstreamConfidenceScore = Json.createValue(1.0f);
+    fineGrainedLineageNode.add("confidenceScore", upstreamConfidenceScore);
+    jsonPatchBuilder.add(
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c1)//urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c1)",
+        fineGrainedLineageNode.build());
+
+    // Initial population test
+    DataJobInputOutput result =
+        dataJobInputOutputTemplate.applyPatch(dataJobInputOutput, jsonPatchBuilder.build());
+    // Hack because Jackson parses values to doubles instead of floats
+    DataMap dataMap = new DataMap();
+    dataMap.put("confidenceScore", 1.0);
+    FineGrainedLineage fineGrainedLineage = new FineGrainedLineage(dataMap);
+    UrnArray urns = new UrnArray();
+    Urn urn1 =
+        UrnUtils.getUrn(
+            "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c1)");
+    urns.add(urn1);
+    UrnArray upstreams = new UrnArray();
+    Urn upstreamUrn =
+        UrnUtils.getUrn(
+            "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c1)");
+    upstreams.add(upstreamUrn);
+    fineGrainedLineage.setDownstreams(urns);
+    fineGrainedLineage.setUpstreams(upstreams);
+    fineGrainedLineage.setTransformOperation("CREATE");
+    fineGrainedLineage.setUpstreamType(FineGrainedLineageUpstreamType.FIELD_SET);
+    fineGrainedLineage.setDownstreamType(FineGrainedLineageDownstreamType.FIELD);
+    assertEquals(result.getFineGrainedLineages().get(0), fineGrainedLineage);
+
+    // Test non-overwrite upstreams and correct confidence score and types w/ overwrite
+    JsonObjectBuilder finegrainedLineageNode2 = Json.createObjectBuilder();
+    finegrainedLineageNode2.add(
+        "upstreamType", Json.createValue(FineGrainedLineageUpstreamType.FIELD_SET.name()));
+    finegrainedLineageNode2.add("confidenceScore", upstreamConfidenceScore);
+    finegrainedLineageNode2.add(
+        "downstreamType", Json.createValue(FineGrainedLineageDownstreamType.FIELD.name()));
+
+    JsonPatchBuilder patchOperations2 = Json.createPatchBuilder();
+    patchOperations2.add(
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)/urn:li:query:someQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)",
+        finegrainedLineageNode2.build());
+
+    JsonValue upstreamConfidenceScore2 = Json.createValue(0.1f);
+    JsonObjectBuilder finegrainedLineageNode3 = Json.createObjectBuilder();
+    finegrainedLineageNode3.add(
+        "upstreamType", Json.createValue(FineGrainedLineageUpstreamType.DATASET.name()));
+    finegrainedLineageNode3.add("confidenceScore", upstreamConfidenceScore2);
+    finegrainedLineageNode3.add(
+        "downstreamType", Json.createValue(FineGrainedLineageDownstreamType.FIELD_SET.name()));
+
+    patchOperations2.add(
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)/urn:li:query:someQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)",
+        finegrainedLineageNode3.build());
+
+    JsonPatch jsonPatch2 = patchOperations2.build();
+
+    DataJobInputOutput result2 = dataJobInputOutputTemplate.applyPatch(result, jsonPatch2);
+    // Hack because Jackson parses values to doubles instead of floats
+    DataMap dataMap2 = new DataMap();
+    dataMap2.put("confidenceScore", 0.1);
+    FineGrainedLineage fineGrainedLineage2 = new FineGrainedLineage(dataMap2);
+    UrnArray urns2 = new UrnArray();
+    Urn urn2 =
+        UrnUtils.getUrn(
+            "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)");
+    urns2.add(urn2);
+    Urn downstreamUrn2 =
+        UrnUtils.getUrn(
+            "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)");
+    UrnArray downstreams2 = new UrnArray();
+    downstreams2.add(downstreamUrn2);
+    fineGrainedLineage2.setUpstreams(urns2);
+    fineGrainedLineage2.setDownstreams(downstreams2);
+    fineGrainedLineage2.setTransformOperation("CREATE");
+    fineGrainedLineage2.setUpstreamType(FineGrainedLineageUpstreamType.DATASET);
+    fineGrainedLineage2.setDownstreamType(FineGrainedLineageDownstreamType.FIELD_SET);
+    fineGrainedLineage2.setQuery(UrnUtils.getUrn("urn:li:query:someQuery"));
+    assertEquals(result2.getFineGrainedLineages().get(1), fineGrainedLineage2);
+
+    // Check different queries
+    JsonObjectBuilder finegrainedLineageNode4 = Json.createObjectBuilder();
+    finegrainedLineageNode4.add(
+        "upstreamType", Json.createValue(FineGrainedLineageUpstreamType.FIELD_SET.name()));
+    finegrainedLineageNode4.add("confidenceScore", upstreamConfidenceScore);
+    finegrainedLineageNode4.add(
+        "downstreamType", Json.createValue(FineGrainedLineageDownstreamType.FIELD.name()));
+
+    JsonPatchBuilder patchOperations3 = Json.createPatchBuilder();
+    patchOperations3.add(
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)/urn:li:query:anotherQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)",
+        finegrainedLineageNode4.build());
+
+    JsonPatch jsonPatch3 = patchOperations3.build();
+    DataJobInputOutput result3 = dataJobInputOutputTemplate.applyPatch(result2, jsonPatch3);
+    // Hack because Jackson parses values to doubles instead of floats
+    DataMap dataMap3 = new DataMap();
+    dataMap3.put("confidenceScore", 1.0);
+    FineGrainedLineage fineGrainedLineage3 = new FineGrainedLineage(dataMap3);
+    UrnArray urns3 = new UrnArray();
+    Urn urn3 =
+        UrnUtils.getUrn(
+            "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)");
+    urns3.add(urn3);
+
+    Urn upstreamUrn3 =
+        UrnUtils.getUrn(
+            "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)");
+    UrnArray upstreamUrns3 = new UrnArray();
+    upstreamUrns3.add(upstreamUrn3);
+    fineGrainedLineage3.setDownstreams(urns3);
+    fineGrainedLineage3.setUpstreams(upstreamUrns3);
+    fineGrainedLineage3.setTransformOperation("CREATE");
+    fineGrainedLineage3.setUpstreamType(FineGrainedLineageUpstreamType.FIELD_SET);
+    fineGrainedLineage3.setDownstreamType(FineGrainedLineageDownstreamType.FIELD);
+    fineGrainedLineage3.setQuery(UrnUtils.getUrn("urn:li:query:anotherQuery"));
+    // Splits into two for different types
+    assertEquals(result3.getFineGrainedLineages().get(2), fineGrainedLineage3);
+
+    // Check different transform types
+    JsonObjectBuilder finegrainedLineageNode5 = Json.createObjectBuilder();
+    finegrainedLineageNode5.add(
+        "upstreamType", Json.createValue(FineGrainedLineageUpstreamType.FIELD_SET.name()));
+    finegrainedLineageNode5.add("confidenceScore", upstreamConfidenceScore);
+    finegrainedLineageNode5.add(
+        "downstreamType", Json.createValue(FineGrainedLineageDownstreamType.FIELD.name()));
+
+    JsonPatchBuilder patchOperations4 = Json.createPatchBuilder();
+    patchOperations4.add(
+        "/fineGrainedLineages/TRANSFORM/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)/urn:li:query:anotherQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)",
+        finegrainedLineageNode5.build());
+    JsonPatch jsonPatch4 = patchOperations4.build();
+
+    DataJobInputOutput result4 = dataJobInputOutputTemplate.applyPatch(result3, jsonPatch4);
+    // Hack because Jackson parses values to doubles instead of floats
+    DataMap dataMap4 = new DataMap();
+    dataMap4.put("confidenceScore", 1.0);
+    FineGrainedLineage fineGrainedLineage4 = new FineGrainedLineage(dataMap4);
+    fineGrainedLineage4.setUpstreams(upstreamUrns3);
+    fineGrainedLineage4.setDownstreams(urns3);
+    fineGrainedLineage4.setTransformOperation("TRANSFORM");
+    fineGrainedLineage4.setUpstreamType(FineGrainedLineageUpstreamType.FIELD_SET);
+    fineGrainedLineage4.setDownstreamType(FineGrainedLineageDownstreamType.FIELD);
+    fineGrainedLineage4.setQuery(UrnUtils.getUrn("urn:li:query:anotherQuery"));
+    // New entry in array because of new transformation type
+    assertEquals(result4.getFineGrainedLineages().get(3), fineGrainedLineage4);
+
+    // Remove
+    JsonPatchBuilder removeOperations = Json.createPatchBuilder();
+    removeOperations.remove(
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c1)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c1)");
+    removeOperations.remove(
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)/urn:li:query:someQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)");
+    removeOperations.remove(
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)/urn:li:query:anotherQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)");
+    removeOperations.remove(
+        "/fineGrainedLineages/TRANSFORM/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)/urn:li:query:anotherQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)");
+
+    JsonPatch removePatch = removeOperations.build();
+    DataJobInputOutput finalResult = dataJobInputOutputTemplate.applyPatch(result4, removePatch);
+    assertEquals(finalResult, dataJobInputOutputTemplate.getDefault());
+  }
+
+  @Test
+  public void testPatchWithFieldWithForwardSlash() throws JsonProcessingException {
+
+    String downstreamUrn =
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c1)";
+    String unescapedUpstreamUrn =
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),slash/column)";
+    String escapedUpstreamUrn =
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),slash~1column)";
+    String lineagePath = downstreamUrn + "//" + escapedUpstreamUrn;
+
+    DataJobInputOutputTemplate dataJobInputOutputTemplate = new DataJobInputOutputTemplate();
+    DataJobInputOutput dataJobInputOutput = dataJobInputOutputTemplate.getDefault();
+    JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder();
+
+    JsonObjectBuilder fineGrainedLineageNode = Json.createObjectBuilder();
+    JsonValue upstreamConfidenceScore = Json.createValue(1.0f);
+    fineGrainedLineageNode.add("confidenceScore", upstreamConfidenceScore);
+
+    jsonPatchBuilder.add(lineagePath, fineGrainedLineageNode.build());
+
+    // Initial population test
+    DataJobInputOutput result =
+        dataJobInputOutputTemplate.applyPatch(dataJobInputOutput, jsonPatchBuilder.build());
+
+    assertEquals(
+        result.getFineGrainedLineages().get(0).getUpstreams().get(0).toString(),
+        unescapedUpstreamUrn);
+  }
+
+  @Test
+  public void testPatchWithFieldWithTilde() throws JsonProcessingException {
+
+    String downstreamUrn =
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c1)";
+    String unescapedUpstreamUrn =
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),tilde~column)";
+    String escapedUpstreamUrn =
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),tilde~0column)";
+    String lineagePath = downstreamUrn + "//" + escapedUpstreamUrn;
+
+    DataJobInputOutputTemplate dataJobInputOutputTemplate = new DataJobInputOutputTemplate();
+    DataJobInputOutput dataJobInputOutput = dataJobInputOutputTemplate.getDefault();
+    JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder();
+
+    JsonObjectBuilder fineGrainedLineageNode = Json.createObjectBuilder();
+    JsonValue upstreamConfidenceScore = Json.createValue(1.0f);
+    fineGrainedLineageNode.add("confidenceScore", upstreamConfidenceScore);
+
+    jsonPatchBuilder.add(lineagePath, fineGrainedLineageNode.build());
+
+    // Initial population test
+    DataJobInputOutput result =
+        dataJobInputOutputTemplate.applyPatch(dataJobInputOutput, jsonPatchBuilder.build());
+    assertEquals(
+        result.getFineGrainedLineages().get(0).getUpstreams().get(0).toString(),
+        unescapedUpstreamUrn);
+  }
+}
diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java
index 5042c35d2f5d4..ab0e7f960251c 100644
--- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java
+++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java
@@ -185,6 +185,29 @@ public void testPatchUpstream() throws Exception {
     // New entry in array because of new transformation type
     assertEquals(result4.getFineGrainedLineages().get(3), fineGrainedLineage4);
 
+    JsonPatchBuilder patchOperations5 = Json.createPatchBuilder();
+    String urn4 =
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,test-bucket/hive/folder_1/folder_2/my_dataset,DEV),c2)";
+    UrnArray downstreamUrns5 = new UrnArray();
+    downstreamUrns5.add(Urn.createFromString(urn4));
+    patchOperations5.add(
+        "/fineGrainedLineages/TRANSFORM/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,test-bucket~1hive~1folder_1~1folder_2~1my_dataset,DEV),c2)/urn:li:query:anotherQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)",
+        finegrainedLineageNode5.build());
+    JsonPatch jsonPatch5 = patchOperations5.build();
+    UpstreamLineage result5 = upstreamLineageTemplate.applyPatch(result4, jsonPatch5);
+    // Hack because Jackson parses values to doubles instead of floats
+    DataMap dataMap5 = new DataMap();
+    dataMap5.put("confidenceScore", 1.0);
+    FineGrainedLineage fineGrainedLineage5 = new FineGrainedLineage(dataMap5);
+    fineGrainedLineage5.setUpstreams(upstreamUrns3);
+    fineGrainedLineage5.setDownstreams(downstreamUrns5);
+    fineGrainedLineage5.setTransformOperation("TRANSFORM");
+    fineGrainedLineage5.setUpstreamType(FineGrainedLineageUpstreamType.FIELD_SET);
+    fineGrainedLineage5.setDownstreamType(FineGrainedLineageDownstreamType.FIELD);
+    fineGrainedLineage5.setQuery(UrnUtils.getUrn("urn:li:query:anotherQuery"));
+    // New entry in array because of new transformation type
+    assertEquals(result5.getFineGrainedLineages().get(4), fineGrainedLineage5);
+
     // Remove
     JsonPatchBuilder removeOperations = Json.createPatchBuilder();
     removeOperations.remove(
@@ -198,6 +221,7 @@ public void testPatchUpstream() throws Exception {
 
     JsonPatch removePatch = removeOperations.build();
     UpstreamLineage finalResult = upstreamLineageTemplate.applyPatch(result4, removePatch);
+
     assertEquals(finalResult, upstreamLineageTemplate.getDefault());
   }
 
@@ -249,8 +273,8 @@ public void testLargePatchStandard() throws Exception {
     UpstreamLineage result = upstreamLineageTemplate.applyPatch(upstreamLineage, jsonPatch);
     long end = System.currentTimeMillis();
     assertTrue(
-        end - start < 10000,
-        String.format("Expected less then 10 seconds patch actual %s ms", end - start));
+        end - start < 20000,
+        String.format("Expected less then 20 seconds patch actual %s ms", end - start));
 
     assertEquals(result.getUpstreams().size(), 187, "Expected 1 less upstream");
     assertEquals(result.getFineGrainedLineages().size(), 607);
@@ -314,4 +338,39 @@ public void testPatchWithFieldWithTilde() throws JsonProcessingException {
         result.getFineGrainedLineages().get(0).getUpstreams().get(0).toString(),
         unescapedUpstreamUrn);
   }
+
+  @Test
+  public void testPatchRemoveWithFields() throws JsonProcessingException {
+
+    String downstreamUrn =
+        "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,~1tmp~1test.parquet,PROD),c1)";
+    String upstreamUrn =
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c1)";
+    String upstreamUrn2 =
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),c2)";
+
+    String lineagePath1 = downstreamUrn + "/NONE/" + upstreamUrn;
+    String lineagePath2 = downstreamUrn + "/NONE/" + upstreamUrn2;
+
+    UpstreamLineageTemplate upstreamLineageTemplate = new UpstreamLineageTemplate();
+    UpstreamLineage upstreamLineage = upstreamLineageTemplate.getDefault();
+    JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder();
+
+    JsonObjectBuilder fineGrainedLineageNode = Json.createObjectBuilder();
+    JsonValue upstreamConfidenceScore = Json.createValue(1.0f);
+    fineGrainedLineageNode.add("confidenceScore", upstreamConfidenceScore);
+
+    jsonPatchBuilder.add(lineagePath1, fineGrainedLineageNode.build());
+    jsonPatchBuilder.add(lineagePath2, fineGrainedLineageNode.build());
+
+    // Initial population test
+    UpstreamLineage result =
+        upstreamLineageTemplate.applyPatch(upstreamLineage, jsonPatchBuilder.build());
+    assertEquals(
+        result.getFineGrainedLineages().get(0).getUpstreams().get(0).toString(), upstreamUrn);
+    assertEquals(
+        result.getFineGrainedLineages().get(0).getUpstreams().get(1).toString(), upstreamUrn2);
+
+    assertEquals(result.getFineGrainedLineages().get(0).getUpstreams().size(), 2);
+  }
 }
diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/FieldPathValidatorTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/FieldPathValidatorTest.java
new file mode 100644
index 0000000000000..bd5912764edce
--- /dev/null
+++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/FieldPathValidatorTest.java
@@ -0,0 +1,233 @@
+package com.linkedin.metadata.aspect.validators;
+
+import static com.linkedin.metadata.Constants.*;
+import static org.mockito.Mockito.*;
+import static org.testng.Assert.*;
+
+import com.linkedin.common.urn.DatasetUrn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.GraphRetriever;
+import com.linkedin.metadata.aspect.RetrieverContext;
+import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
+import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import com.linkedin.metadata.aspect.validation.CreateIfNotExistsValidator;
+import com.linkedin.metadata.aspect.validation.FieldPathValidator;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import com.linkedin.schema.EditableSchemaFieldInfo;
+import com.linkedin.schema.EditableSchemaFieldInfoArray;
+import com.linkedin.schema.EditableSchemaMetadata;
+import com.linkedin.schema.SchemaField;
+import com.linkedin.schema.SchemaFieldArray;
+import com.linkedin.schema.SchemaFieldDataType;
+import com.linkedin.schema.SchemaMetadata;
+import com.linkedin.schema.StringType;
+import com.linkedin.test.metadata.aspect.TestEntityRegistry;
+import com.linkedin.test.metadata.aspect.batch.TestMCP;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+public class FieldPathValidatorTest {
+
+  private static final AspectPluginConfig validatorConfig =
+      AspectPluginConfig.builder()
+          .supportedOperations(
+              Arrays.stream(ChangeType.values())
+                  .map(Objects::toString)
+                  .collect(Collectors.toList()))
+          .className(CreateIfNotExistsValidator.class.getName())
+          .supportedEntityAspectNames(List.of(AspectPluginConfig.EntityAspectName.ALL))
+          .enabled(true)
+          .build();
+  private EntityRegistry entityRegistry;
+  private RetrieverContext mockRetrieverContext;
+  private static final DatasetUrn TEST_DATASET_URN;
+  private final FieldPathValidator test = new FieldPathValidator().setConfig(validatorConfig);
+
+  static {
+    try {
+      TEST_DATASET_URN =
+          DatasetUrn.createFromUrn(
+              UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,test,PROD)"));
+    } catch (URISyntaxException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @BeforeTest
+  public void init() {
+    entityRegistry = new TestEntityRegistry();
+    AspectRetriever mockAspectRetriever = mock(AspectRetriever.class);
+    when(mockAspectRetriever.getEntityRegistry()).thenReturn(entityRegistry);
+    GraphRetriever mockGraphRetriever = mock(GraphRetriever.class);
+    mockRetrieverContext = mock(RetrieverContext.class);
+    when(mockRetrieverContext.getAspectRetriever()).thenReturn(mockAspectRetriever);
+    when(mockRetrieverContext.getGraphRetriever()).thenReturn(mockGraphRetriever);
+  }
+
+  @Test
+  public void testValidateNonDuplicatedSchemaFieldPath() {
+    final SchemaMetadata schema = getMockSchemaMetadataAspect(false);
+    assertEquals(
+        test.validateProposed(
+                Set.of(
+                    TestMCP.builder()
+                        .changeType(ChangeType.UPSERT)
+                        .urn(TEST_DATASET_URN)
+                        .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType()))
+                        .aspectSpec(
+                            entityRegistry
+                                .getEntitySpec(TEST_DATASET_URN.getEntityType())
+                                .getAspectSpec(SCHEMA_METADATA_ASPECT_NAME))
+                        .recordTemplate(schema)
+                        .build()),
+                mockRetrieverContext)
+            .count(),
+        0);
+  }
+
+  @Test
+  public void testValidateDuplicatedSchemaFieldPath() {
+    final SchemaMetadata schema = getMockSchemaMetadataAspect(true);
+
+    assertEquals(
+        test.validateProposed(
+                Set.of(
+                    TestMCP.builder()
+                        .changeType(ChangeType.UPSERT)
+                        .urn(TEST_DATASET_URN)
+                        .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType()))
+                        .aspectSpec(
+                            entityRegistry
+                                .getEntitySpec(TEST_DATASET_URN.getEntityType())
+                                .getAspectSpec(SCHEMA_METADATA_ASPECT_NAME))
+                        .recordTemplate(schema)
+                        .build()),
+                mockRetrieverContext)
+            .count(),
+        1);
+  }
+
+  @Test
+  public void testValidateNonDuplicatedEditableSchemaFieldPath() {
+    final EditableSchemaMetadata schema = getMockEditableSchemaMetadataAspect(false);
+    assertEquals(
+        test.validateProposed(
+                Set.of(
+                    TestMCP.builder()
+                        .changeType(ChangeType.UPSERT)
+                        .urn(TEST_DATASET_URN)
+                        .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType()))
+                        .aspectSpec(
+                            entityRegistry
+                                .getEntitySpec(TEST_DATASET_URN.getEntityType())
+                                .getAspectSpec(EDITABLE_SCHEMA_METADATA_ASPECT_NAME))
+                        .recordTemplate(schema)
+                        .build()),
+                mockRetrieverContext)
+            .count(),
+        0);
+  }
+
+  @Test
+  public void testValidateDuplicatedEditableSchemaFieldPath() {
+    final EditableSchemaMetadata schema = getMockEditableSchemaMetadataAspect(true);
+
+    assertEquals(
+        test.validateProposed(
+                Set.of(
+                    TestMCP.builder()
+                        .changeType(ChangeType.UPSERT)
+                        .urn(TEST_DATASET_URN)
+                        .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType()))
+                        .aspectSpec(
+                            entityRegistry
+                                .getEntitySpec(TEST_DATASET_URN.getEntityType())
+                                .getAspectSpec(EDITABLE_SCHEMA_METADATA_ASPECT_NAME))
+                        .recordTemplate(schema)
+                        .build()),
+                mockRetrieverContext)
+            .count(),
+        1);
+  }
+
+  @Test
+  public void testEmptySchemaFieldPath() {
+    final SchemaMetadata schema = getMockSchemaMetadataAspect(false, "");
+    TestMCP testItem =
+        TestMCP.builder()
+            .changeType(ChangeType.UPSERT)
+            .urn(TEST_DATASET_URN)
+            .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType()))
+            .aspectSpec(
+                entityRegistry
+                    .getEntitySpec(TEST_DATASET_URN.getEntityType())
+                    .getAspectSpec(SCHEMA_METADATA_ASPECT_NAME))
+            .recordTemplate(schema)
+            .build();
+
+    Set<AspectValidationException> exceptions =
+        test.validateProposed(Set.of(testItem), mockRetrieverContext).collect(Collectors.toSet());
+
+    assertEquals(
+        exceptions,
+        Set.of(
+            AspectValidationException.forItem(
+                testItem, "SchemaMetadata aspect has empty field path.")));
+  }
+
+  private static SchemaMetadata getMockSchemaMetadataAspect(boolean duplicateFields) {
+    return getMockSchemaMetadataAspect(duplicateFields, null);
+  }
+
+  private static SchemaMetadata getMockSchemaMetadataAspect(
+      boolean duplicateFields, @Nullable String fieldPath) {
+    List<SchemaField> fields = new ArrayList<>();
+    fields.add(
+        new SchemaField()
+            .setType(
+                new SchemaFieldDataType()
+                    .setType(SchemaFieldDataType.Type.create(new StringType())))
+            .setNullable(false)
+            .setNativeDataType("string")
+            .setFieldPath(fieldPath == null ? "test" : fieldPath));
+
+    if (duplicateFields) {
+      fields.add(
+          new SchemaField()
+              .setType(
+                  new SchemaFieldDataType()
+                      .setType(SchemaFieldDataType.Type.create(new StringType())))
+              .setNullable(false)
+              .setNativeDataType("string")
+              .setFieldPath(fieldPath == null ? "test" : fieldPath));
+    }
+
+    return new SchemaMetadata()
+        .setPlatform(TEST_DATASET_URN.getPlatformEntity())
+        .setFields(new SchemaFieldArray(fields));
+  }
+
+  private static EditableSchemaMetadata getMockEditableSchemaMetadataAspect(
+      boolean duplicateFields) {
+
+    List<EditableSchemaFieldInfo> fields = new ArrayList<>();
+    fields.add(new EditableSchemaFieldInfo().setFieldPath("test"));
+
+    if (duplicateFields) {
+      fields.add(new EditableSchemaFieldInfo().setFieldPath("test"));
+    }
+
+    return new EditableSchemaMetadata()
+        .setEditableSchemaFieldInfo(new EditableSchemaFieldInfoArray(fields));
+  }
+}
diff --git a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java
index 65705f15022b6..98a6d59004a92 100644
--- a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java
+++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java
@@ -5,7 +5,7 @@
 import com.linkedin.data.DataMap;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.entity.Aspect;
-import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.aspect.SystemAspect;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.mxe.SystemMetadata;
@@ -22,7 +22,7 @@
 import javax.annotation.Nonnull;
 import org.mockito.Mockito;
 
-public class MockAspectRetriever implements AspectRetriever {
+public class MockAspectRetriever implements CachingAspectRetriever {
   private final Map<Urn, Map<String, Aspect>> data;
   private final Map<Urn, Map<String, SystemAspect>> systemData = new HashMap<>();
 
diff --git a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java
index 7dd889c48b874..6643a9de58562 100644
--- a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java
+++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java
@@ -4,10 +4,12 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.MCLItem;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.mxe.MetadataChangeLog;
+import java.util.Objects;
 import javax.annotation.Nonnull;
 import lombok.Builder;
 import lombok.Getter;
@@ -29,4 +31,23 @@ public class TestMCL implements MCLItem {
   public String getAspectName() {
     return getAspectSpec().getName();
   }
+
+  @Override
+  public boolean isDatabaseDuplicateOf(BatchItem other) {
+    return equals(other);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    TestMCL testMCL = (TestMCL) o;
+    return Objects.equals(metadataChangeLog, testMCL.metadataChangeLog);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hashCode(metadataChangeLog);
+  }
 }
diff --git a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java
index e562390a959a7..d7dd1fab2b6ac 100644
--- a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java
+++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java
@@ -6,6 +6,7 @@
 
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
+import com.linkedin.data.template.DataTemplateUtil;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.ReadItem;
@@ -20,7 +21,9 @@
 import com.linkedin.test.metadata.aspect.TestEntityRegistry;
 import java.net.URISyntaxException;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -138,6 +141,42 @@ public Map<String, String> getHeaders() {
             mcp ->
                 mcp.getHeaders().entrySet().stream()
                     .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))
-        .orElse(headers);
+        .orElse(headers != null ? headers : Collections.emptyMap());
+  }
+
+  @Override
+  public boolean isDatabaseDuplicateOf(BatchItem other) {
+    return equals(other);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    TestMCP testMCP = (TestMCP) o;
+    return urn.equals(testMCP.urn)
+        && DataTemplateUtil.areEqual(recordTemplate, testMCP.recordTemplate)
+        && Objects.equals(systemAspect, testMCP.systemAspect)
+        && Objects.equals(previousSystemAspect, testMCP.previousSystemAspect)
+        && Objects.equals(auditStamp, testMCP.auditStamp)
+        && Objects.equals(changeType, testMCP.changeType)
+        && Objects.equals(metadataChangeProposal, testMCP.metadataChangeProposal);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = urn.hashCode();
+    result = 31 * result + Objects.hashCode(recordTemplate);
+    result = 31 * result + Objects.hashCode(systemAspect);
+    result = 31 * result + Objects.hashCode(previousSystemAspect);
+    result = 31 * result + Objects.hashCode(auditStamp);
+    result = 31 * result + Objects.hashCode(changeType);
+    result = 31 * result + Objects.hashCode(metadataChangeProposal);
+    return result;
   }
 }
diff --git a/gradle/coverage/java-coverage.gradle b/gradle/coverage/java-coverage.gradle
new file mode 100644
index 0000000000000..17260c1a30978
--- /dev/null
+++ b/gradle/coverage/java-coverage.gradle
@@ -0,0 +1,31 @@
+apply plugin: "jacoco"
+
+jacoco {
+  toolVersion = "0.8.12"
+}
+
+/*
+These need to run after evaluation since jacoco plugin alters the test task based on the included test
+lib dependencies defined in each subproject (junit or testNG)
+*/
+afterEvaluate {
+  test {
+    finalizedBy jacocoTestReport
+  }
+
+  jacocoTestReport {
+    dependsOn test
+    reports {
+      xml {
+        required = true
+        /*
+        Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one
+        folder will minimize the time spent searching through the full source tree.
+        */
+        outputLocation = rootProject.layout.buildDirectory.file("coverage-reports/jacoco-${project.name}.xml")
+      }
+      csv.required = false
+      html.required = false
+    }
+  }
+}
\ No newline at end of file
diff --git a/gradle/coverage/python-coverage.gradle b/gradle/coverage/python-coverage.gradle
new file mode 100644
index 0000000000000..23d6e37387ed8
--- /dev/null
+++ b/gradle/coverage/python-coverage.gradle
@@ -0,0 +1,18 @@
+//coverage related args to be passed to pytest
+ext.get_coverage_args = { test_name = "" ->
+
+  def coverage_file_name = "coverage-${project.name}${test_name}.xml"
+
+  /*
+  Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one folder
+  will minimize the time spent searching through the full source tree.
+  */
+  def base_path = "${rootProject.buildDir}/coverage-reports"
+
+  /*
+  --cov=src was added via setup.cfg in many of the python projects but for some reason, was not getting picked up
+  consistently, so adding it explicitly. Centralizing these params would make it easier to adjust them for all python
+  projects (with overrides being in the sub-project build script.)
+   */
+  return "--cov-report xml:${base_path}/${coverage_file_name} --cov=src"
+}
\ No newline at end of file
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index ccebba7710dea..a4b76b9530d66 100644
Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index bdc9a83b1e652..22286c90de3d1 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,8 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip
+distributionSha256Sum=89d4e70e4e84e2d2dfbb63e4daa53e21b25017cc70c37e4eea31ee51fb15098a
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
index 79a61d421cc4e..f5feea6d6b116 100755
--- a/gradlew
+++ b/gradlew
@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -83,10 +85,9 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,10 +134,13 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
@@ -144,7 +148,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
@@ -152,7 +156,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -197,11 +201,15 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \
diff --git a/gradlew.bat b/gradlew.bat
index 6689b85beecde..9b42019c7915b 100644
--- a/gradlew.bat
+++ b/gradlew.bat
@@ -13,6 +13,8 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 @rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
 
 @if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
@@ -43,11 +45,11 @@ set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
 if %ERRORLEVEL% equ 0 goto execute
 
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 
@@ -57,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
 if exist "%JAVA_EXE%" goto execute
 
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 
diff --git a/ingestion-scheduler/build.gradle b/ingestion-scheduler/build.gradle
index 9505ec57aa858..fb1cfd4e27a4b 100644
--- a/ingestion-scheduler/build.gradle
+++ b/ingestion-scheduler/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'java'
 }
 
+apply from: "../gradle/coverage/java-coverage.gradle"
+
 dependencies {
     implementation project(path: ':metadata-models')
     implementation project(path: ':metadata-io')
diff --git a/li-utils/build.gradle b/li-utils/build.gradle
index 975cd2bccccf3..e00fcc2854612 100644
--- a/li-utils/build.gradle
+++ b/li-utils/build.gradle
@@ -1,8 +1,11 @@
+
 plugins {
     id 'java-library'
     id 'pegasus'
 }
 
+apply from: "../gradle/coverage/java-coverage.gradle"
+
 
 dependencies {
     api spec.product.pegasus.data
diff --git a/li-utils/src/main/java/com/datahub/util/validator/RelationshipValidator.java b/li-utils/src/main/java/com/datahub/util/validator/RelationshipValidator.java
index c96f1c4fb7313..4a0e219031fef 100644
--- a/li-utils/src/main/java/com/datahub/util/validator/RelationshipValidator.java
+++ b/li-utils/src/main/java/com/datahub/util/validator/RelationshipValidator.java
@@ -1,14 +1,9 @@
 package com.datahub.util.validator;
 
-import com.linkedin.common.urn.Urn;
-import com.linkedin.data.DataList;
-import com.linkedin.data.DataMap;
 import com.linkedin.data.schema.RecordDataSchema;
 import com.linkedin.data.schema.UnionDataSchema;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.data.template.UnionTemplate;
-import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import javax.annotation.Nonnull;
@@ -63,8 +58,6 @@ public static void validateRelationshipSchema(@Nonnull RecordDataSchema schema)
                   "Relationship '%s' contains a field '%s' that makes use of a disallowed type '%s'.",
                   className, field.getName(), field.getType().getType());
             });
-
-    validatePairings(schema);
   }
 
   /**
@@ -109,59 +102,4 @@ public static void validateRelationshipUnionSchema(
           relationshipClassName);
     }
   }
-
-  private static void validatePairings(@Nonnull RecordDataSchema schema) {
-
-    final String className = schema.getBindingName();
-
-    Map<String, Object> properties = schema.getProperties();
-    if (!properties.containsKey("pairings")) {
-      ValidationUtils.invalidSchema(
-          "Relationship '%s' must contain a 'pairings' property", className);
-    }
-
-    DataList pairings = (DataList) properties.get("pairings");
-    Set<Pair> registeredPairs = new HashSet<>();
-    pairings.stream()
-        .forEach(
-            obj -> {
-              DataMap map = (DataMap) obj;
-              if (!map.containsKey("source") || !map.containsKey("destination")) {
-                ValidationUtils.invalidSchema(
-                    "Relationship '%s' contains an invalid 'pairings' item. "
-                        + "Each item must contain a 'source' and 'destination' properties.",
-                    className);
-              }
-
-              String sourceUrn = map.getString("source");
-              if (!isValidUrnClass(sourceUrn)) {
-                ValidationUtils.invalidSchema(
-                    "Relationship '%s' contains an invalid item in 'pairings'. %s is not a valid URN class name.",
-                    className, sourceUrn);
-              }
-
-              String destinationUrn = map.getString("destination");
-              if (!isValidUrnClass(destinationUrn)) {
-                ValidationUtils.invalidSchema(
-                    "Relationship '%s' contains an invalid item in 'pairings'. %s is not a valid URN class name.",
-                    className, destinationUrn);
-              }
-
-              Pair pair = new Pair(sourceUrn, destinationUrn);
-              if (registeredPairs.contains(pair)) {
-                ValidationUtils.invalidSchema(
-                    "Relationship '%s' contains a repeated 'pairings' item (%s, %s)",
-                    className, sourceUrn, destinationUrn);
-              }
-              registeredPairs.add(pair);
-            });
-  }
-
-  private static boolean isValidUrnClass(String className) {
-    try {
-      return Urn.class.isAssignableFrom(Class.forName(className));
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException(e);
-    }
-  }
 }
diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java
index e085a5876a42b..09f873ebf7bc9 100644
--- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java
+++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java
@@ -10,6 +10,7 @@ public class Constants {
   public static final String INTERNAL_DELEGATED_FOR_ACTOR_HEADER_NAME = "X-DataHub-Delegated-For";
   public static final String INTERNAL_DELEGATED_FOR_ACTOR_TYPE = "X-DataHub-Delegated-For-";
 
+  public static final String URN_LI_PREFIX = "urn:li:";
   public static final String DATAHUB_ACTOR = "urn:li:corpuser:datahub"; // Super user.
   public static final String SYSTEM_ACTOR =
       "urn:li:corpuser:__datahub_system"; // DataHub internal service principal.
@@ -50,6 +51,7 @@ public class Constants {
   // App sources
   public static final String UI_SOURCE = "ui";
   public static final String SYSTEM_UPDATE_SOURCE = "systemUpdate";
+  public static final String METADATA_TESTS_SOURCE = "metadataTests";
 
   /** Entities */
   public static final String CORP_USER_ENTITY_NAME = "corpuser";
@@ -106,6 +108,8 @@ public class Constants {
   // Common
   public static final String OWNERSHIP_ASPECT_NAME = "ownership";
 
+  public static final String TIMESTAMP_MILLIS = "timestampMillis";
+
   public static final String INSTITUTIONAL_MEMORY_ASPECT_NAME = "institutionalMemory";
   public static final String DATA_PLATFORM_INSTANCE_ASPECT_NAME = "dataPlatformInstance";
   public static final String BROWSE_PATHS_ASPECT_NAME = "browsePaths";
@@ -277,7 +281,7 @@ public class Constants {
 
   // ExternalRoleMetadata
   public static final String ROLE_ENTITY_NAME = "role";
-  public static final String ACCESS_DATASET_ASPECT_NAME = "access";
+  public static final String ACCESS_ASPECT_NAME = "access";
   public static final String ROLE_KEY = "roleKey";
   public static final String ROLE_PROPERTIES_ASPECT_NAME = "roleProperties";
   public static final String ROLE_ACTORS_ASPECT_NAME = "actors";
@@ -319,6 +323,13 @@ public class Constants {
   public static final String EXECUTION_REQUEST_INPUT_ASPECT_NAME = "dataHubExecutionRequestInput";
   public static final String EXECUTION_REQUEST_SIGNAL_ASPECT_NAME = "dataHubExecutionRequestSignal";
   public static final String EXECUTION_REQUEST_RESULT_ASPECT_NAME = "dataHubExecutionRequestResult";
+  public static final String EXECUTION_REQUEST_STATUS_RUNNING = "RUNNING";
+  public static final String EXECUTION_REQUEST_STATUS_FAILURE = "FAILURE";
+  public static final String EXECUTION_REQUEST_STATUS_SUCCESS = "SUCCESS";
+  public static final String EXECUTION_REQUEST_STATUS_TIMEOUT = "TIMEOUT";
+  public static final String EXECUTION_REQUEST_STATUS_CANCELLED = "CANCELLED";
+  public static final String EXECUTION_REQUEST_STATUS_ABORTED = "ABORTED";
+  public static final String EXECUTION_REQUEST_STATUS_DUPLICATE = "DUPLICATE";
 
   // DataHub Access Token
   public static final String ACCESS_TOKEN_KEY_ASPECT_NAME = "dataHubAccessTokenKey";
@@ -354,6 +365,8 @@ public class Constants {
   // Structured Property
   public static final String STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME = "propertyDefinition";
   public static final String STRUCTURED_PROPERTY_KEY_ASPECT_NAME = "structuredPropertyKey";
+  public static final String STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME =
+      "structuredPropertySettings";
 
   // Form
   public static final String FORM_INFO_ASPECT_NAME = "formInfo";
@@ -396,6 +409,8 @@ public class Constants {
   /** User Status */
   public static final String CORP_USER_STATUS_ACTIVE = "ACTIVE";
 
+  public static final String CORP_USER_STATUS_SUSPENDED = "SUSPENDED";
+
   /** Task Runs */
   public static final String DATA_PROCESS_INSTANCE_ENTITY_NAME = "dataProcessInstance";
 
@@ -451,5 +466,13 @@ public class Constants {
   // DAO
   public static final long LATEST_VERSION = 0;
 
+  // Logging MDC
+  public static final String MDC_ENTITY_URN = "entityUrn";
+  public static final String MDC_ASPECT_NAME = "aspectName";
+  public static final String MDC_ENTITY_TYPE = "entityType";
+  public static final String MDC_CHANGE_TYPE = "changeType";
+
+  public static final String RESTLI_SUCCESS = "success";
+
   private Constants() {}
 }
diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/UrnUtils.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/UrnUtils.java
index 89f0cd8fbc979..0a2400badfc62 100644
--- a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/UrnUtils.java
+++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/UrnUtils.java
@@ -27,28 +27,6 @@ public static DatasetUrn toDatasetUrn(
         new DataPlatformUrn(platformName), datasetName, FabricType.valueOf(origin.toUpperCase()));
   }
 
-  /**
-   * Convert fabric String to FabricType
-   *
-   * @param fabric PROD, CORP, EI, DEV, LIT, PRIME
-   * @return FabricType
-   */
-  @Nonnull
-  public static FabricType toFabricType(@Nonnull String fabric) {
-    switch (fabric.toUpperCase()) {
-      case "PROD":
-        return FabricType.PROD;
-      case "CORP":
-        return FabricType.CORP;
-      case "EI":
-        return FabricType.EI;
-      case "DEV":
-        return FabricType.DEV;
-      default:
-        throw new IllegalArgumentException("Unsupported Fabric Type: " + fabric);
-    }
-  }
-
   public static Urn getUrn(String urnStr) {
     try {
       return Urn.createFromString(urnStr);
diff --git a/metadata-auth/auth-api/build.gradle b/metadata-auth/auth-api/build.gradle
index acc1af7e2e3ad..9a833dacf7fb1 100644
--- a/metadata-auth/auth-api/build.gradle
+++ b/metadata-auth/auth-api/build.gradle
@@ -2,12 +2,13 @@ plugins {
   id("com.palantir.git-version") apply false
 }
 
-apply plugin: 'com.github.johnrengelman.shadow'
+apply plugin: 'com.gradleup.shadow'
 apply plugin: 'java-library'
 apply plugin: 'signing'
 apply plugin: 'maven-publish'
 apply plugin: 'io.codearte.nexus-staging'
 apply from: '../../metadata-integration/java/versioning.gradle'
+apply from: '../../gradle/coverage/java-coverage.gradle'
 
 jar {
   archiveClassifier = "lib"
diff --git a/metadata-dao-impl/kafka-producer/build.gradle b/metadata-dao-impl/kafka-producer/build.gradle
index 2df15309810db..d3ba031817afe 100644
--- a/metadata-dao-impl/kafka-producer/build.gradle
+++ b/metadata-dao-impl/kafka-producer/build.gradle
@@ -1,5 +1,7 @@
 apply plugin: 'java'
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(':metadata-events:mxe-avro')
   implementation project(':metadata-events:mxe-registration')
diff --git a/metadata-events/mxe-schemas/build.gradle b/metadata-events/mxe-schemas/build.gradle
index ab0ea8b649e9d..6dfe69a420242 100644
--- a/metadata-events/mxe-schemas/build.gradle
+++ b/metadata-events/mxe-schemas/build.gradle
@@ -25,7 +25,7 @@ task copyOriginalAvsc(type: Copy, dependsOn: generateAvroSchema) {
 }
 
 task renameNamespace(type: Exec, dependsOn: copyOriginalAvsc) {
-  commandLine 'sh', './rename-namespace.sh'
+  commandLine 'bash', './rename-namespace.sh'
 }
 
 build.dependsOn renameNamespace
@@ -34,4 +34,4 @@ clean {
   project.delete('src/main/pegasus')
   project.delete('src/mainGeneratedAvroSchema/avro')
   project.delete('src/renamed/avro')
-}
\ No newline at end of file
+}
diff --git a/metadata-events/mxe-utils-avro/build.gradle b/metadata-events/mxe-utils-avro/build.gradle
index 860ced6af2581..f893fadb8e2df 100644
--- a/metadata-events/mxe-utils-avro/build.gradle
+++ b/metadata-events/mxe-utils-avro/build.gradle
@@ -3,6 +3,8 @@ plugins {
   id 'pegasus'
 }
 
+apply from: "../../gradle/coverage/java-coverage.gradle"
+
 dependencies {
   api project(':metadata-events:mxe-avro')
   api project(':metadata-models')
diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle
index 9506609a10044..68a35c0dfc417 100644
--- a/metadata-ingestion-modules/airflow-plugin/build.gradle
+++ b/metadata-ingestion-modules/airflow-plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'base'
 }
 
+apply from: "../../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -11,7 +13,7 @@ if (!project.hasProperty("extra_pip_requirements")) {
     ext.extra_pip_requirements = ""
 }
 if (!project.hasProperty("extra_pip_extras")) {
-    ext.extra_pip_extras = "plugin-v2"
+    ext.extra_pip_extras = ""
 }
 // If extra_pip_extras is non-empty, we need to add a comma to the beginning of the string.
 if (extra_pip_extras != "") {
@@ -97,7 +99,7 @@ task testQuick(type: Exec, dependsOn: installTest) {
   inputs.files(project.fileTree(dir: "tests/"))
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+    "pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py
index 8461628cfa334..2693aab0700da 100644
--- a/metadata-ingestion-modules/airflow-plugin/setup.py
+++ b/metadata-ingestion-modules/airflow-plugin/setup.py
@@ -24,8 +24,8 @@ def get_long_description():
 
 base_requirements = {
     f"acryl-datahub[datahub-rest]{_self_pin}",
-    # Actual dependencies.
-    "apache-airflow >= 2.0.2",
+    # We require Airflow 2.3.x, since we need the new DAG listener API.
+    "apache-airflow>=2.3.0",
 }
 
 plugins: Dict[str, Set[str]] = {
@@ -44,12 +44,13 @@ def get_long_description():
         # We remain restrictive on the versions allowed here to prevent
         # us from being broken by backwards-incompatible changes in the
         # underlying package.
-        "openlineage-airflow>=1.2.0,<=1.18.0",
+        "openlineage-airflow>=1.2.0,<=1.25.0",
     },
 }
 
-# Include datahub-rest in the base requirements.
+# Require some plugins by default.
 base_requirements.update(plugins["datahub-rest"])
+base_requirements.update(plugins["plugin-v2"])
 
 
 mypy_stubs = {
@@ -96,7 +97,7 @@ def get_long_description():
     *plugins["datahub-kafka"],
     f"acryl-datahub[testing-utils]{_self_pin}",
     # Extra requirements for loading our test dags.
-    "apache-airflow[snowflake]>=2.0.2",
+    "apache-airflow[snowflake,amazon]>=2.0.2",
     # A collection of issues we've encountered:
     # - Connexion's new version breaks Airflow:
     #   See https://github.com/apache/airflow/issues/35234.
@@ -109,6 +110,11 @@ def get_long_description():
     "apache-airflow-providers-sqlite",
 }
 per_version_test_requirements = {
+    "test-airflow23": {
+        "pendulum<3.0",
+        "Flask-Session<0.6.0",
+        "connexion<3.0",
+    },
     "test-airflow24": {
         "pendulum<3.0",
         "Flask-Session<0.6.0",
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py
index c1e2dd4cc422d..d86a46e042e8f 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py
@@ -46,7 +46,7 @@ def get_task_inlets(operator: "Operator") -> List:
         return operator._inlets  # type: ignore[attr-defined, union-attr]
     if hasattr(operator, "get_inlet_defs"):
         return operator.get_inlet_defs()  # type: ignore[attr-defined]
-    return operator.inlets
+    return operator.inlets or []
 
 
 def get_task_outlets(operator: "Operator") -> List:
@@ -56,7 +56,7 @@ def get_task_outlets(operator: "Operator") -> List:
         return operator._outlets  # type: ignore[attr-defined, union-attr]
     if hasattr(operator, "get_outlet_defs"):
         return operator.get_outlet_defs()
-    return operator.outlets
+    return operator.outlets or []
 
 
 __all__ = [
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py
index 8deba22a107ce..c4964712cf9f7 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py
@@ -3,7 +3,8 @@
 
 import datahub.emitter.mce_builder as builder
 from airflow.configuration import conf
-from datahub.configuration.common import ConfigModel
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from pydantic.fields import Field
 
 if TYPE_CHECKING:
     from datahub_airflow_plugin.hooks.datahub import DatahubGenericHook
@@ -56,6 +57,11 @@ class DatahubLineageConfig(ConfigModel):
     # Makes extraction of jinja-templated fields more accurate.
     render_templates: bool = True
 
+    dag_filter_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="regex patterns for DAGs to ingest",
+    )
+
     log_level: Optional[str] = None
     debug_emitter: bool = False
 
@@ -93,6 +99,9 @@ def get_lineage_config() -> DatahubLineageConfig:
     datajob_url_link = conf.get(
         "datahub", "datajob_url_link", fallback=DatajobUrl.TASKINSTANCE.value
     )
+    dag_filter_pattern = AllowDenyPattern.parse_raw(
+        conf.get("datahub", "dag_filter_str", fallback='{"allow": [".*"]}')
+    )
 
     return DatahubLineageConfig(
         enabled=enabled,
@@ -109,4 +118,5 @@ def get_lineage_config() -> DatahubLineageConfig:
         disable_openlineage_plugin=disable_openlineage_plugin,
         datajob_url_link=datajob_url_link,
         render_templates=render_templates,
+        dag_filter_pattern=dag_filter_pattern,
     )
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py
index 7d35791bf1db4..69de61aced0a5 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py
@@ -8,7 +8,6 @@
 
 OL_SCHEME_TWEAKS = {
     "sqlserver": "mssql",
-    "trino": "presto",
     "awsathena": "athena",
 }
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py
index de0d4f8711f53..28d5775f61f54 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py
@@ -50,7 +50,6 @@ def __init__(self):
             "BigQueryOperator",
             "BigQueryExecuteQueryOperator",
             # Athena also does something similar.
-            "AthenaOperator",
             "AWSAthenaOperator",
             # Additional types that OL doesn't support. This is only necessary because
             # on older versions of Airflow, these operators don't inherit from SQLExecuteQueryOperator.
@@ -59,6 +58,8 @@ def __init__(self):
         for operator in _sql_operator_overrides:
             self.task_to_extractor.extractors[operator] = GenericSqlExtractor
 
+        self.task_to_extractor.extractors["AthenaOperator"] = AthenaOperatorExtractor
+
         self.task_to_extractor.extractors[
             "BigQueryInsertJobOperator"
         ] = BigQueryInsertJobOperatorExtractor
@@ -276,6 +277,27 @@ def extract(self) -> Optional[TaskMetadata]:
         )
 
 
+class AthenaOperatorExtractor(BaseExtractor):
+    def extract(self) -> Optional[TaskMetadata]:
+        from airflow.providers.amazon.aws.operators.athena import (
+            AthenaOperator,  # type: ignore
+        )
+
+        operator: "AthenaOperator" = self.operator
+        sql = operator.query
+        if not sql:
+            self.log.warning("No query found in AthenaOperator")
+            return None
+
+        return _parse_sql_into_task_metadata(
+            self,
+            sql,
+            platform="athena",
+            default_database=None,
+            default_schema=self.operator.database,
+        )
+
+
 def _snowflake_default_schema(self: "SnowflakeExtractor") -> Optional[str]:
     if hasattr(self.operator, "schema") and self.operator.schema is not None:
         return self.operator.schema
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
index b818b76de9f7f..640991a90a1d2 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
@@ -9,6 +9,7 @@
 
 import airflow
 import datahub.emitter.mce_builder as builder
+from airflow.models import Variable
 from airflow.models.serialized_dag import SerializedDagModel
 from datahub.api.entities.datajob import DataJob
 from datahub.api.entities.dataprocess.dataprocess_instance import InstanceRunResult
@@ -74,10 +75,12 @@ def hookimpl(f: _F) -> _F:  # type: ignore[misc] # noqa: F811
     "1",
 )
 _RUN_IN_THREAD_TIMEOUT = float(
-    os.getenv("DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD_TIMEOUT", 15)
+    os.getenv("DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD_TIMEOUT", 10)
 )
 _DATAHUB_CLEANUP_DAG = "Datahub_Cleanup"
 
+KILL_SWITCH_VARIABLE_NAME = "datahub_airflow_plugin_disable_listener"
+
 
 def get_airflow_plugin_listener() -> Optional["DataHubListener"]:
     # Using globals instead of functools.lru_cache to make testing easier.
@@ -102,6 +105,7 @@ def get_airflow_plugin_listener() -> Optional["DataHubListener"]:
                     "capture_tags": plugin_config.capture_tags_info,
                     "capture_ownership": plugin_config.capture_ownership_info,
                     "enable_extractors": plugin_config.enable_extractors,
+                    "render_templates": plugin_config.render_templates,
                     "disable_openlineage_plugin": plugin_config.disable_openlineage_plugin,
                 },
             )
@@ -363,6 +367,12 @@ def _extract_lineage(
                     redact_with_exclusions(v)
                 )
 
+    def check_kill_switch(self):
+        if Variable.get(KILL_SWITCH_VARIABLE_NAME, "false").lower() == "true":
+            logger.debug("DataHub listener disabled by kill switch")
+            return True
+        return False
+
     @hookimpl
     @run_in_thread
     def on_task_instance_running(
@@ -371,6 +381,8 @@ def on_task_instance_running(
         task_instance: "TaskInstance",
         session: "Session",  # This will always be QUEUED
     ) -> None:
+        if self.check_kill_switch():
+            return
         self._set_log_level()
 
         # This if statement mirrors the logic in https://github.com/OpenLineage/OpenLineage/pull/508.
@@ -383,9 +395,13 @@ def on_task_instance_running(
             return
 
         logger.debug(
-            f"DataHub listener got notification about task instance start for {task_instance.task_id}"
+            f"DataHub listener got notification about task instance start for {task_instance.task_id} of dag {task_instance.dag_id}"
         )
 
+        if not self.config.dag_filter_pattern.allowed(task_instance.dag_id):
+            logger.debug(f"DAG {task_instance.dag_id} is not allowed by the pattern")
+            return
+
         if self.config.render_templates:
             task_instance = _render_templates(task_instance)
 
@@ -449,6 +465,9 @@ def on_task_instance_running(
             f"DataHub listener finished processing notification about task instance start for {task_instance.task_id}"
         )
 
+        self.materialize_iolets(datajob)
+
+    def materialize_iolets(self, datajob: DataJob) -> None:
         if self.config.materialize_iolets:
             for outlet in datajob.outlets:
                 reported_time: int = int(time.time() * 1000)
@@ -492,6 +511,10 @@ def on_task_instance_finish(
 
         dag: "DAG" = task.dag  # type: ignore[assignment]
 
+        if not self.config.dag_filter_pattern.allowed(dag.dag_id):
+            logger.debug(f"DAG {dag.dag_id} is not allowed by the pattern")
+            return
+
         datajob = AirflowGenerator.generate_datajob(
             cluster=self.config.cluster,
             task=task,
@@ -532,6 +555,9 @@ def on_task_instance_finish(
     def on_task_instance_success(
         self, previous_state: None, task_instance: "TaskInstance", session: "Session"
     ) -> None:
+        if self.check_kill_switch():
+            return
+
         self._set_log_level()
 
         logger.debug(
@@ -547,6 +573,9 @@ def on_task_instance_success(
     def on_task_instance_failed(
         self, previous_state: None, task_instance: "TaskInstance", session: "Session"
     ) -> None:
+        if self.check_kill_switch():
+            return
+
         self._set_log_level()
 
         logger.debug(
@@ -562,6 +591,9 @@ def on_task_instance_failed(
     def on_dag_start(self, dag_run: "DagRun") -> None:
         dag = dag_run.dag
         if not dag:
+            logger.warning(
+                f"DataHub listener could not find DAG for {dag_run.dag_id} - {dag_run.run_id}. Dag won't be captured"
+            )
             return
 
         dataflow = AirflowGenerator.generate_dataflow(
@@ -569,6 +601,7 @@ def on_dag_start(self, dag_run: "DagRun") -> None:
             dag=dag,
         )
         dataflow.emit(self.emitter, callback=self._make_emit_callback())
+        logger.debug(f"Emitted DataHub DataFlow: {dataflow}")
 
         event: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper(
             entityUrn=str(dataflow.urn), aspect=StatusClass(removed=False)
@@ -683,14 +716,21 @@ def on_dag_start(self, dag_run: "DagRun") -> None:
         @hookimpl
         @run_in_thread
         def on_dag_run_running(self, dag_run: "DagRun", msg: str) -> None:
+            if self.check_kill_switch():
+                return
+
             self._set_log_level()
 
             logger.debug(
                 f"DataHub listener got notification about dag run start for {dag_run.dag_id}"
             )
 
-            self.on_dag_start(dag_run)
+            assert dag_run.dag_id
+            if not self.config.dag_filter_pattern.allowed(dag_run.dag_id):
+                logger.debug(f"DAG {dag_run.dag_id} is not allowed by the pattern")
+                return
 
+            self.on_dag_start(dag_run)
             self.emitter.flush()
 
     # TODO: Add hooks for on_dag_run_success, on_dag_run_failed -> call AirflowGenerator.complete_dataflow
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py
new file mode 100644
index 0000000000000..d72ba67c23cd7
--- /dev/null
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py
@@ -0,0 +1,35 @@
+"""This example DAG demonstrates how to create and use a DataHubGraph client."""
+
+from datetime import timedelta
+
+import pendulum
+from airflow.decorators import dag, task
+from datahub.ingestion.graph.client import DataHubGraph, RemovedStatusFilter
+
+from datahub_airflow_plugin.hooks.datahub import DatahubRestHook
+
+
+@dag(
+    schedule_interval=timedelta(days=1),
+    start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
+    catchup=False,
+)
+def datahub_graph_usage_sample_dag():
+    @task()
+    def use_the_graph():
+        graph: DataHubGraph = DatahubRestHook("my_datahub_rest_conn_id").make_graph()
+        graph.test_connection()
+
+        # Example usage: Find all soft-deleted BigQuery DEV entities
+        # in DataHub, and hard delete them.
+        for urn in graph.get_urns_by_filter(
+            platform="bigquery",
+            env="DEV",
+            status=RemovedStatusFilter.ONLY_SOFT_DELETED,
+        ):
+            graph.hard_delete_entity(urn)
+
+    use_the_graph()
+
+
+datahub_graph_usage_sample_dag()
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/hooks/datahub.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/hooks/datahub.py
index b60f20c5bf8b2..5f4d787fb893d 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/hooks/datahub.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/hooks/datahub.py
@@ -14,6 +14,7 @@
     from datahub.emitter.kafka_emitter import DatahubKafkaEmitter
     from datahub.emitter.rest_emitter import DataHubRestEmitter
     from datahub.emitter.synchronized_file_emitter import SynchronizedFileEmitter
+    from datahub.ingestion.graph.client import DataHubGraph
     from datahub.ingestion.sink.datahub_kafka import KafkaSinkConfig
 
 
@@ -94,6 +95,9 @@ def make_emitter(self) -> "DataHubRestEmitter":
             host, token, **extra_args
         )
 
+    def make_graph(self) -> "DataHubGraph":
+        return self.make_emitter().to_graph()
+
     def emit(
         self,
         items: Sequence[
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/athena_operator.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/athena_operator.py
new file mode 100644
index 0000000000000..96cdacbbad37d
--- /dev/null
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/athena_operator.py
@@ -0,0 +1,43 @@
+from datetime import datetime
+
+from airflow import DAG
+from airflow.providers.amazon.aws.operators.athena import AthenaOperator
+
+ATHENA_COST_TABLE = "costs"
+ATHENA_PROCESSED_TABLE = "processed_costs"
+
+
+def _fake_athena_execute(*args, **kwargs):
+    pass
+
+
+with DAG(
+    "athena_operator",
+    start_date=datetime(2023, 1, 1),
+    schedule_interval=None,
+    catchup=False,
+) as dag:
+    # HACK: We don't want to send real requests to Athena. As a workaround,
+    # we can simply monkey-patch the operator.
+    AthenaOperator.execute = _fake_athena_execute  # type: ignore
+
+    transform_cost_table = AthenaOperator(
+        aws_conn_id="my_aws",
+        task_id="transform_cost_table",
+        database="athena_db",
+        query="""
+        CREATE OR REPLACE TABLE {{ params.out_table_name }} AS
+        SELECT
+            id,
+            month,
+            total_cost,
+            area,
+            total_cost / area as cost_per_area
+        FROM {{ params.in_table_name }}
+        """,
+        params={
+            "in_table_name": ATHENA_COST_TABLE,
+            "out_table_name": ATHENA_PROCESSED_TABLE,
+        },
+        output_location="s3://athena-results-bucket/",
+    )
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py
new file mode 100644
index 0000000000000..a805a2219d142
--- /dev/null
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py
@@ -0,0 +1,34 @@
+from datetime import datetime
+
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+
+from datahub_airflow_plugin.entities import Dataset, Urn
+
+with DAG(
+    "dag_to_skip",
+    start_date=datetime(2023, 1, 1),
+    schedule_interval=None,
+    catchup=False,
+) as dag:
+    task1 = BashOperator(
+        task_id="dag_to_skip_task_1",
+        dag=dag,
+        bash_command="echo 'dag_to_skip_task_1'",
+        inlets=[
+            Dataset(platform="snowflake", name="mydb.schema.tableA"),
+            Urn(
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
+            ),
+            Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"),
+        ],
+        outlets=[Dataset("snowflake", "mydb.schema.tableD")],
+    )
+
+    task2 = BashOperator(
+        task_id="dag_to_skip_task_2",
+        dag=dag,
+        bash_command="echo 'dag_to_skip_task_2'",
+    )
+
+    task1 >> task2
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
index fd10f858d00fb..4c21b7ed4000d 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
@@ -14,7 +14,7 @@
                 "fileloc": "<fileloc>",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
-                "tags": "None",
+                "tags": "[]",
                 "timezone": "Timezone('UTC')"
             },
             "externalUrl": "http://airflow.example.com/tree?dag_id=basic_iolets",
@@ -83,7 +83,7 @@
                 "execution_timeout": "None",
                 "sla": "None",
                 "task_id": "'run_data_task'",
-                "trigger_rule": "'all_success'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "[]",
                 "inlets": "[]",
@@ -246,6 +246,46 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "depends_on_past": "False",
+                "email": "None",
+                "label": "'run_data_task'",
+                "execution_timeout": "None",
+                "sla": "None",
+                "task_id": "'run_data_task'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
+                "wait_for_downstream": "False",
+                "downstream_task_ids": "[]",
+                "inlets": "[]",
+                "outlets": "[]"
+            },
+            "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task",
+            "name": "run_data_task",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)",
@@ -402,16 +442,16 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets",
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
                 "orchestrator": "airflow",
                 "dag_id": "basic_iolets",
                 "task_id": "run_data_task"
             },
-            "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets",
+            "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
             "name": "basic_iolets_run_data_task_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1717180290951,
+                "time": 1733529136396,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -544,7 +584,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1717180290951,
+            "timestampMillis": 1733529136396,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"
@@ -561,7 +601,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1717180291140,
+            "timestampMillis": 1733529137385,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
index 5c5be6848fd83..b6ab1ff9120f2 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
@@ -14,7 +14,7 @@
                 "fileloc": "<fileloc>",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
-                "tags": "None",
+                "tags": "[]",
                 "timezone": "Timezone('UTC')"
             },
             "externalUrl": "http://airflow.example.com/tree?dag_id=simple_dag",
@@ -84,7 +84,7 @@
                 "execution_timeout": "None",
                 "sla": "None",
                 "task_id": "'task_1'",
-                "trigger_rule": "'all_success'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "['run_another_data_task']",
                 "inlets": "[]",
@@ -205,6 +205,46 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "depends_on_past": "False",
+                "email": "None",
+                "label": "'task_1'",
+                "execution_timeout": "None",
+                "sla": "None",
+                "task_id": "'task_1'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
+                "wait_for_downstream": "False",
+                "downstream_task_ids": "['run_another_data_task']",
+                "inlets": "[]",
+                "outlets": "[]"
+            },
+            "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1",
+            "name": "task_1",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)",
@@ -319,16 +359,16 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "2",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag",
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
                 "orchestrator": "airflow",
                 "dag_id": "simple_dag",
                 "task_id": "task_1"
             },
-            "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag",
+            "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_task_1_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1717180227827,
+                "time": 1733528983395,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -419,7 +459,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1717180227827,
+            "timestampMillis": 1733528983395,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"
@@ -436,7 +476,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1717180228022,
+            "timestampMillis": 1733528984355,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"
@@ -449,6 +489,42 @@
         }
     }
 },
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "dataFlowInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "_access_control": "None",
+                "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
+                "fileloc": "<fileloc>",
+                "is_paused_upon_creation": "None",
+                "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
+                "tags": "[]",
+                "timezone": "Timezone('UTC')"
+            },
+            "externalUrl": "http://airflow.example.com/tree?dag_id=simple_dag",
+            "name": "simple_dag",
+            "description": "A simple DAG that runs a few fake data tasks.",
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataFlow",
     "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)",
@@ -498,7 +574,7 @@
                 "execution_timeout": "None",
                 "sla": "None",
                 "task_id": "'run_another_data_task'",
-                "trigger_rule": "'all_success'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "[]",
                 "inlets": "[]",
@@ -575,6 +651,46 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),run_another_data_task)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "depends_on_past": "False",
+                "email": "None",
+                "label": "'run_another_data_task'",
+                "execution_timeout": "None",
+                "sla": "None",
+                "task_id": "'run_another_data_task'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
+                "wait_for_downstream": "False",
+                "downstream_task_ids": "[]",
+                "inlets": "[]",
+                "outlets": "[]"
+            },
+            "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=run_another_data_task",
+            "name": "run_another_data_task",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),run_another_data_task)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),run_another_data_task)",
@@ -645,16 +761,16 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag",
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
                 "orchestrator": "airflow",
                 "dag_id": "simple_dag",
                 "task_id": "run_another_data_task"
             },
-            "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag",
+            "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_run_another_data_task_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1717180231676,
+                "time": 1733528992448,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -679,7 +795,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1717180231676,
+            "timestampMillis": 1733528992448,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"
@@ -696,7 +812,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1717180231824,
+            "timestampMillis": 1733528993380,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json
new file mode 100644
index 0000000000000..baa738fef7b5f
--- /dev/null
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json
@@ -0,0 +1,672 @@
+[
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "dataFlowInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "_access_control": "None",
+                "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
+                "fileloc": "<fileloc>",
+                "is_paused_upon_creation": "None",
+                "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
+                "tags": "[]",
+                "timezone": "Timezone('UTC')"
+            },
+            "externalUrl": "http://airflow.example.com/tree?dag_id=athena_operator",
+            "name": "athena_operator",
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:airflow",
+                    "type": "DEVELOPER",
+                    "source": {
+                        "type": "SERVICE"
+                    }
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:airflow"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "athena_operator"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "depends_on_past": "False",
+                "email": "None",
+                "label": "'transform_cost_table'",
+                "execution_timeout": "None",
+                "sla": "None",
+                "task_id": "'transform_cost_table'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
+                "wait_for_downstream": "False",
+                "downstream_task_ids": "[]",
+                "inlets": "[]",
+                "outlets": "[]",
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE OR REPLACE TABLE processed_costs AS\\n        SELECT\\n            id,\\n            month,\\n            total_cost,\\n            area,\\n            total_cost / area as cost_per_area\\n        FROM costs\\n        \"}"
+            },
+            "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table",
+            "name": "transform_cost_table",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)"
+            ],
+            "outputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)"
+            ],
+            "inputDatajobs": [],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.processed_costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:airflow",
+                    "type": "DEVELOPER",
+                    "source": {
+                        "type": "SERVICE"
+                    }
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:airflow"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "run_id": "manual_run_test",
+                "duration": "<duration>",
+                "start_date": "<start_date>",
+                "end_date": "<end_date>",
+                "execution_date": "2023-09-27 21:34:38+00:00",
+                "try_number": "0",
+                "max_tries": "0",
+                "external_executor_id": "None",
+                "state": "running",
+                "operator": "AthenaOperator",
+                "priority_weight": "1",
+                "log_url": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs",
+                "orchestrator": "airflow",
+                "dag_id": "athena_operator",
+                "task_id": "transform_cost_table"
+            },
+            "externalUrl": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs",
+            "name": "athena_operator_transform_cost_table_manual_run_test",
+            "type": "BATCH_AD_HOC",
+            "created": {
+                "time": 1732719433576,
+                "actor": "urn:li:corpuser:datahub"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRelationships",
+    "aspect": {
+        "json": {
+            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+            "upstreamInstances": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceInput",
+    "aspect": {
+        "json": {
+            "inputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)"
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceOutput",
+    "aspect": {
+        "json": {
+            "outputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)"
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.processed_costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1732719433576,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "STARTED",
+            "attempt": 1
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "operation",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1732719433736,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "actor": "urn:li:corpuser:airflow",
+            "operationType": "CREATE",
+            "lastUpdatedTimestamp": 1732719433736
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "depends_on_past": "False",
+                "email": "None",
+                "label": "'transform_cost_table'",
+                "execution_timeout": "None",
+                "sla": "None",
+                "task_id": "'transform_cost_table'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
+                "wait_for_downstream": "False",
+                "downstream_task_ids": "[]",
+                "inlets": "[]",
+                "outlets": "[]",
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE OR REPLACE TABLE processed_costs AS\\n        SELECT\\n            id,\\n            month,\\n            total_cost,\\n            area,\\n            total_cost / area as cost_per_area\\n        FROM costs\\n        \"}"
+            },
+            "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table",
+            "name": "transform_cost_table",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)"
+            ],
+            "outputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)"
+            ],
+            "inputDatajobs": [],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.processed_costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:airflow",
+                    "type": "DEVELOPER",
+                    "source": {
+                        "type": "SERVICE"
+                    }
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:airflow"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1732719433747,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "COMPLETE",
+            "result": {
+                "type": "SUCCESS",
+                "nativeResultType": "airflow"
+            }
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator_no_dag_listener.json
new file mode 100644
index 0000000000000..c53825a9979e3
--- /dev/null
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator_no_dag_listener.json
@@ -0,0 +1,672 @@
+[
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "dataFlowInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "_access_control": "None",
+                "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
+                "fileloc": "<fileloc>",
+                "is_paused_upon_creation": "None",
+                "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
+                "tags": "[]",
+                "timezone": "Timezone('UTC')"
+            },
+            "externalUrl": "http://airflow.example.com/tree?dag_id=athena_operator",
+            "name": "athena_operator",
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:airflow",
+                    "type": "DEVELOPER",
+                    "source": {
+                        "type": "SERVICE"
+                    }
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:airflow"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "athena_operator"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "depends_on_past": "False",
+                "email": "None",
+                "label": "'transform_cost_table'",
+                "execution_timeout": "None",
+                "sla": "None",
+                "task_id": "'transform_cost_table'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
+                "wait_for_downstream": "False",
+                "downstream_task_ids": "[]",
+                "inlets": "[]",
+                "outlets": "[]",
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE OR REPLACE TABLE processed_costs AS\\n        SELECT\\n            id,\\n            month,\\n            total_cost,\\n            area,\\n            total_cost / area as cost_per_area\\n        FROM costs\\n        \"}"
+            },
+            "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table",
+            "name": "transform_cost_table",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)"
+            ],
+            "outputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)"
+            ],
+            "inputDatajobs": [],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.processed_costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:airflow",
+                    "type": "DEVELOPER",
+                    "source": {
+                        "type": "SERVICE"
+                    }
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:airflow"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "run_id": "manual_run_test",
+                "duration": "<duration>",
+                "start_date": "<start_date>",
+                "end_date": "<end_date>",
+                "execution_date": "2023-09-27 21:34:38+00:00",
+                "try_number": "0",
+                "max_tries": "0",
+                "external_executor_id": "None",
+                "state": "running",
+                "operator": "AthenaOperator",
+                "priority_weight": "1",
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=athena_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "athena_operator",
+                "task_id": "transform_cost_table"
+            },
+            "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=athena_operator&map_index=-1",
+            "name": "athena_operator_transform_cost_table_manual_run_test",
+            "type": "BATCH_AD_HOC",
+            "created": {
+                "time": 1733121901482,
+                "actor": "urn:li:corpuser:datahub"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRelationships",
+    "aspect": {
+        "json": {
+            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+            "upstreamInstances": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceInput",
+    "aspect": {
+        "json": {
+            "inputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)"
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceOutput",
+    "aspect": {
+        "json": {
+            "outputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)"
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.processed_costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1733121901482,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "STARTED",
+            "attempt": 1
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "operation",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1733121901625,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "actor": "urn:li:corpuser:airflow",
+            "operationType": "CREATE",
+            "lastUpdatedTimestamp": 1733121901625
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "depends_on_past": "False",
+                "email": "None",
+                "label": "'transform_cost_table'",
+                "execution_timeout": "None",
+                "sla": "None",
+                "task_id": "'transform_cost_table'",
+                "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
+                "wait_for_downstream": "False",
+                "downstream_task_ids": "[]",
+                "inlets": "[]",
+                "outlets": "[]",
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE OR REPLACE TABLE processed_costs AS\\n        SELECT\\n            id,\\n            month,\\n            total_cost,\\n            area,\\n            total_cost / area as cost_per_area\\n        FROM costs\\n        \"}"
+            },
+            "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table",
+            "name": "transform_cost_table",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)"
+            ],
+            "outputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)"
+            ],
+            "inputDatajobs": [],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:athena",
+            "name": "athena_db.processed_costs",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:airflow",
+                    "type": "DEVELOPER",
+                    "source": {
+                        "type": "SERVICE"
+                    }
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:airflow"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1733121901675,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "COMPLETE",
+            "result": {
+                "type": "SUCCESS",
+                "nativeResultType": "airflow"
+            }
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py
index 37cd3b792d535..3b2c9140e4632 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py
@@ -8,9 +8,11 @@
 import random
 import signal
 import subprocess
+import textwrap
 import time
 from typing import Any, Iterator, Sequence
 
+import packaging.version
 import pytest
 import requests
 import tenacity
@@ -19,6 +21,7 @@
 from datahub.testing.compare_metadata_json import assert_metadata_files_equal
 
 from datahub_airflow_plugin._airflow_shims import (
+    AIRFLOW_VERSION,
     HAS_AIRFLOW_DAG_LISTENER_API,
     HAS_AIRFLOW_LISTENER_API,
     HAS_AIRFLOW_STANDALONE_CMD,
@@ -33,6 +36,8 @@
 DAGS_FOLDER = pathlib.Path(__file__).parent / "dags"
 GOLDENS_FOLDER = pathlib.Path(__file__).parent / "goldens"
 
+DAG_TO_SKIP_INGESTION = "dag_to_skip"
+
 
 @dataclasses.dataclass
 class AirflowInstance:
@@ -108,6 +113,66 @@ def _wait_for_dag_finish(
         raise NotReadyError(f"DAG has not finished yet: {dag_run['state']}")
 
 
+@tenacity.retry(
+    reraise=True,
+    wait=tenacity.wait_fixed(1),
+    stop=tenacity.stop_after_delay(90),
+    retry=tenacity.retry_if_exception_type(NotReadyError),
+)
+def _wait_for_dag_to_load(airflow_instance: AirflowInstance, dag_id: str) -> None:
+    print("Checking if DAG was loaded")
+    res = airflow_instance.session.get(
+        url=f"{airflow_instance.airflow_url}/api/v1/dags",
+        timeout=5,
+    )
+    res.raise_for_status()
+
+    if len(list(filter(lambda x: x["dag_id"] == dag_id, res.json()["dags"]))) == 0:
+        raise NotReadyError("DAG was not loaded yet")
+
+
+def _dump_dag_logs(airflow_instance: AirflowInstance, dag_id: str) -> None:
+    # Get the dag run info
+    res = airflow_instance.session.get(
+        f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns", timeout=5
+    )
+    res.raise_for_status()
+    dag_run = res.json()["dag_runs"][0]
+    dag_run_id = dag_run["dag_run_id"]
+
+    # List the tasks in the dag run
+    res = airflow_instance.session.get(
+        f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances",
+        timeout=5,
+    )
+    res.raise_for_status()
+    task_instances = res.json()["task_instances"]
+
+    # Sort tasks by start_date to maintain execution order
+    task_instances.sort(key=lambda x: x["start_date"] or "")
+
+    print(f"\nTask execution order for DAG {dag_id}:")
+    for task in task_instances:
+        task_id = task["task_id"]
+        state = task["state"]
+        try_number = task.get("try_number", 1)
+
+        task_header = f"Task: {task_id} (State: {state}; Try: {try_number})"
+
+        # Get logs for the task's latest try number
+        try:
+            res = airflow_instance.session.get(
+                f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}"
+                f"/taskInstances/{task_id}/logs/{try_number}",
+                params={"full_content": "true"},
+                timeout=5,
+            )
+            res.raise_for_status()
+            print(f"\n=== {task_header} ===\n{textwrap.indent(res.text, '    ')}")
+        except Exception as e:
+            print(f"Failed to fetch logs for {task_header}: {e}")
+
+
 @contextlib.contextmanager
 def _run_airflow(
     tmp_path: pathlib.Path,
@@ -140,6 +205,7 @@ def _run_airflow(
         # Configure the datahub plugin and have it write the MCPs to a file.
         "AIRFLOW__CORE__LAZY_LOAD_PLUGINS": "False" if is_v1 else "True",
         "AIRFLOW__DATAHUB__CONN_ID": datahub_connection_name,
+        "AIRFLOW__DATAHUB__DAG_FILTER_STR": f'{{ "deny": ["{DAG_TO_SKIP_INGESTION}"] }}',
         f"AIRFLOW_CONN_{datahub_connection_name.upper()}": Connection(
             conn_id="datahub_file_default",
             conn_type="datahub-file",
@@ -160,6 +226,15 @@ def _run_airflow(
                 "insecure_mode": "true",
             },
         ).get_uri(),
+        "AIRFLOW_CONN_MY_AWS": Connection(
+            conn_id="my_aws",
+            conn_type="aws",
+            extra={
+                "region_name": "us-east-1",
+                "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE",
+                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+            },
+        ).get_uri(),
         "AIRFLOW_CONN_MY_SQLITE": Connection(
             conn_id="my_sqlite",
             conn_type="sqlite",
@@ -169,6 +244,7 @@ def _run_airflow(
         # Note that we could also disable the RUN_IN_THREAD entirely,
         # but I want to minimize the difference between CI and prod.
         "DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD_TIMEOUT": "30",
+        "DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN": "true" if is_v1 else "false",
         # Convenience settings.
         "AIRFLOW__DATAHUB__LOG_LEVEL": "DEBUG",
         "AIRFLOW__DATAHUB__DEBUG_EMITTER": "True",
@@ -276,17 +352,18 @@ class DagTestCase:
 test_cases = [
     DagTestCase("simple_dag"),
     DagTestCase("basic_iolets"),
+    DagTestCase("dag_to_skip", v2_only=True),
     DagTestCase("snowflake_operator", success=False, v2_only=True),
     DagTestCase("sqlite_operator", v2_only=True),
     DagTestCase("custom_operator_dag", v2_only=True),
     DagTestCase("datahub_emitter_operator_jinja_template_dag", v2_only=True),
+    DagTestCase("athena_operator", v2_only=True),
 ]
 
 
 @pytest.mark.parametrize(
     ["golden_filename", "test_case", "is_v1"],
     [
-        # On Airflow <= 2.2, test plugin v1.
         *[
             pytest.param(
                 f"v1_{test_case.dag_id}",
@@ -294,8 +371,8 @@ class DagTestCase:
                 True,
                 id=f"v1_{test_case.dag_id}",
                 marks=pytest.mark.skipif(
-                    HAS_AIRFLOW_LISTENER_API,
-                    reason="Not testing plugin v1 on newer Airflow versions",
+                    AIRFLOW_VERSION >= packaging.version.parse("2.4.0"),
+                    reason="We only test the v1 plugin on Airflow 2.3",
                 ),
             )
             for test_case in test_cases
@@ -316,10 +393,18 @@ class DagTestCase:
                     if HAS_AIRFLOW_DAG_LISTENER_API
                     else f"v2_{test_case.dag_id}_no_dag_listener"
                 ),
-                marks=pytest.mark.skipif(
-                    not HAS_AIRFLOW_LISTENER_API,
-                    reason="Cannot test plugin v2 without the Airflow plugin listener API",
-                ),
+                marks=[
+                    pytest.mark.skipif(
+                        not HAS_AIRFLOW_LISTENER_API,
+                        reason="Cannot test plugin v2 without the Airflow plugin listener API",
+                    ),
+                    pytest.mark.skipif(
+                        AIRFLOW_VERSION < packaging.version.parse("2.4.0"),
+                        reason="We skip testing the v2 plugin on Airflow 2.3 because it causes flakiness in the custom properties. "
+                        "Ideally we'd just fix these, but given that Airflow 2.3 is EOL and likely going to be deprecated "
+                        "soon anyways, it's not worth the effort.",
+                    ),
+                ],
             )
             for test_case in test_cases
         ],
@@ -351,6 +436,7 @@ def test_airflow_plugin(
         tmp_path, dags_folder=DAGS_FOLDER, is_v1=is_v1
     ) as airflow_instance:
         print(f"Running DAG {dag_id}...")
+        _wait_for_dag_to_load(airflow_instance, dag_id)
         subprocess.check_call(
             [
                 "airflow",
@@ -373,20 +459,29 @@ def test_airflow_plugin(
         print("Sleeping for a few seconds to let the plugin finish...")
         time.sleep(10)
 
-    _sanitize_output_file(airflow_instance.metadata_file)
+        try:
+            _dump_dag_logs(airflow_instance, dag_id)
+        except Exception as e:
+            print(f"Failed to dump DAG logs: {e}")
 
-    check_golden_file(
-        pytestconfig=pytestconfig,
-        output_path=airflow_instance.metadata_file,
-        golden_path=golden_path,
-        ignore_paths=[
-            # TODO: If we switched to Git urls, maybe we could get this to work consistently.
-            r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['datahub_sql_parser_error'\]",
-            r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['openlineage_.*'\]",
-            r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['log_url'\]",
-            r"root\[\d+\]\['aspect'\]\['json'\]\['externalUrl'\]",
-        ],
-    )
+    if dag_id == DAG_TO_SKIP_INGESTION:
+        # Verify that no MCPs were generated.
+        assert not os.path.exists(airflow_instance.metadata_file)
+    else:
+        _sanitize_output_file(airflow_instance.metadata_file)
+
+        check_golden_file(
+            pytestconfig=pytestconfig,
+            output_path=airflow_instance.metadata_file,
+            golden_path=golden_path,
+            ignore_paths=[
+                # TODO: If we switched to Git urls, maybe we could get this to work consistently.
+                r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['datahub_sql_parser_error'\]",
+                r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['openlineage_.*'\]",
+                r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['log_url'\]",
+                r"root\[\d+\]\['aspect'\]\['json'\]\['externalUrl'\]",
+            ],
+        )
 
 
 def _sanitize_output_file(output_path: pathlib.Path) -> None:
diff --git a/metadata-ingestion-modules/airflow-plugin/tox.ini b/metadata-ingestion-modules/airflow-plugin/tox.ini
index 9e0a30df6fcbd..b310ec84248f1 100644
--- a/metadata-ingestion-modules/airflow-plugin/tox.ini
+++ b/metadata-ingestion-modules/airflow-plugin/tox.ini
@@ -4,25 +4,34 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py38-airflow21, py38-airflow22, py310-airflow24, py310-airflow26, py310-airflow27, py310-airflow28, py311-airflow29
+envlist = py38-airflow23, py310-airflow24, py310-airflow26, py310-airflow27, py310-airflow28, py311-airflow29, py311-airflow210
 
 [testenv]
 use_develop = true
-extras = dev,integration-tests,plugin-v1
+extras =
+    dev
+    integration-tests
+    plugin-v1
+    plugin-v2
+    # For Airflow 2.3 and 2.4, add a few extra requirements.
+    airflow23: test-airflow23
+    airflow24: test-airflow24
+
 deps =
     # This should be kept in sync with the Github Actions matrix.
     -e ../../metadata-ingestion/
     # Airflow version
-    airflow21: apache-airflow~=2.1.0
-    airflow22: apache-airflow~=2.2.0
+    airflow23: apache-airflow~=2.3.0
     airflow24: apache-airflow~=2.4.0
     airflow26: apache-airflow~=2.6.0
     airflow27: apache-airflow~=2.7.0
     airflow28: apache-airflow~=2.8.0
     airflow29: apache-airflow~=2.9.0
+    airflow210: apache-airflow~=2.10.0
 
     # Respect the Airflow constraints files.
-    # We can't make ourselves work with the constraints of Airflow < 2.3.
+    # We can't make ourselves work with the constraints of Airflow <= 2.3.
+    ; py38-airflow23: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.3.4/constraints-3.8.txt
     # The Airflow 2.4 constraints file requires a version of the sqlite provider whose
     # hook type is missing the `conn_name_attr` property.
     ; py310-airflow24: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.4.3/constraints-3.10.txt
@@ -30,6 +39,7 @@ deps =
     py310-airflow27: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt
     py310-airflow28: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt
     py311-airflow29: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt
+    py311-airflow210: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt
 
     # Before pinning to the constraint files, we previously left the dependencies
     # more open. There were a number of packages for which this caused issues.
@@ -52,11 +62,3 @@ deps =
     ; airflow24,airflow26,airflow27,airflow28: Flask-Session<0.6.0
 commands =
     pytest --cov-append {posargs}
-
-# For Airflow 2.4+, add the plugin-v2 extra.
-[testenv:py310-airflow24]
-extras = dev,integration-tests,plugin-v2,test-airflow24
-
-[testenv:py310-airflow{26,27,28},py311-airflow{29}]
-extras = dev,integration-tests,plugin-v2
-
diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle
index 0f11af9ca83d7..0d57bb5bfdff7 100644
--- a/metadata-ingestion-modules/dagster-plugin/build.gradle
+++ b/metadata-ingestion-modules/dagster-plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'base'
 }
 
+apply from: "../../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -84,7 +86,7 @@ task testQuick(type: Exec, dependsOn: installDevTest) {
   outputs.dir("${venv_name}")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+    "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
 task buildWheel(type: Exec, dependsOn: [environmentSetup]) {
diff --git a/metadata-ingestion-modules/dagster-plugin/setup.py b/metadata-ingestion-modules/dagster-plugin/setup.py
index ad9d01e6412f3..22c15497bd807 100644
--- a/metadata-ingestion-modules/dagster-plugin/setup.py
+++ b/metadata-ingestion-modules/dagster-plugin/setup.py
@@ -13,14 +13,6 @@ def get_long_description():
     return pathlib.Path(os.path.join(root, "README.md")).read_text()
 
 
-rest_common = {"requests", "requests_file"}
-
-sqlglot_lib = {
-    # Using an Acryl fork of sqlglot.
-    # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main?expand=1
-    "acryl-sqlglot[rs]==24.0.1.dev7",
-}
-
 _version: str = package_metadata["__version__"]
 _self_pin = (
     f"=={_version}"
@@ -32,11 +24,7 @@ def get_long_description():
     # Actual dependencies.
     "dagster >= 1.3.3",
     "dagit >= 1.3.3",
-    *rest_common,
-    # Ignoring the dependency below because it causes issues with the vercel built wheel install
-    # f"acryl-datahub[datahub-rest]{_self_pin}",
-    "acryl-datahub[datahub-rest]",
-    *sqlglot_lib,
+    f"acryl-datahub[datahub-rest,sql-parser]{_self_pin}",
 }
 
 mypy_stubs = {
@@ -132,7 +120,7 @@ def get_long_description():
     ],
     # Package info.
     zip_safe=False,
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     package_dir={"": "src"},
     packages=setuptools.find_namespace_packages(where="./src"),
     entry_points=entry_points,
diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py
index df9d0fc423fcf..a87f490f2d947 100644
--- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py
+++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py
@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from datetime import datetime, timezone
 from logging import Logger
 from typing import Any, Callable, Dict, List, NamedTuple, Optional, Sequence, Set
 from urllib.parse import urlsplit
@@ -12,7 +13,14 @@
     TableSchemaMetadataValue,
 )
 from dagster._core.execution.stats import RunStepKeyStatsSnapshot, StepEventStatus
-from dagster._core.snap import JobSnapshot
+from datahub.sql_parsing.sqlglot_utils import get_query_fingerprint
+
+try:
+    from dagster._core.snap import JobSnapshot  # type: ignore[attr-defined]
+except ImportError:
+    # Import changed since Dagster 1.8.12  to this -> https://github.com/dagster-io/dagster/commit/29a37d1f0260cfd112849633d1096ffc916d6c95
+    from dagster._core.snap import JobSnap as JobSnapshot
+
 from dagster._core.snap.node import OpDefSnap
 from dagster._core.storage.dagster_run import DagsterRun, DagsterRunStatsSnapshot
 from datahub.api.entities.datajob import DataFlow, DataJob
@@ -26,13 +34,22 @@
     make_data_platform_urn,
     make_dataplatform_instance_urn,
     make_tag_urn,
+    make_ts_millis,
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
 from datahub.metadata._schema_classes import (
+    AuditStampClass,
     BrowsePathEntryClass,
     BrowsePathsV2Class,
     GlobalTagsClass,
+    QueryLanguageClass,
+    QueryPropertiesClass,
+    QuerySourceClass,
+    QueryStatementClass,
+    QuerySubjectClass,
+    QuerySubjectsClass,
     TagAssociationClass,
 )
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
@@ -57,6 +74,7 @@
     SubTypesClass,
     UpstreamClass,
 )
+from datahub.metadata.urns import CorpUserUrn
 from datahub.specific.dataset import DatasetPatchBuilder
 from datahub.utilities.urns._urn_base import Urn
 from datahub.utilities.urns.data_flow_urn import DataFlowUrn
@@ -67,6 +85,8 @@
 
 DAGSTER_PLATFORM = "dagster"
 
+_DEFAULT_USER_URN = CorpUserUrn("_ingestion")
+
 
 class Constant:
     """
@@ -78,12 +98,6 @@ class Constant:
     # Default config constants
     DEFAULT_DATAHUB_REST_URL = "http://localhost:8080"
 
-    # Environment variable contants
-    DATAHUB_REST_URL = "DATAHUB_REST_URL"
-    DATAHUB_ENV = "DATAHUB_ENV"
-    DATAHUB_PLATFORM_INSTANCE = "DATAHUB_PLATFORM_INSTANCE"
-    DAGSTER_UI_URL = "DAGSTER_UI_URL"
-
     # Datahub inputs/outputs constant
     DATAHUB_INPUTS = "datahub.inputs"
     DATAHUB_OUTPUTS = "datahub.outputs"
@@ -154,7 +168,6 @@ class DatasetLineage(NamedTuple):
 
 class DatahubDagsterSourceConfig(DatasetSourceConfigMixin):
     datahub_client_config: DatahubClientConfig = pydantic.Field(
-        default=DatahubClientConfig(),
         description="Datahub client config",
     )
 
@@ -213,6 +226,16 @@ class DatahubDagsterSourceConfig(DatasetSourceConfigMixin):
         description="Whether to materialize asset dependency in DataHub. It emits a datasetKey for each dependencies. Default is False.",
     )
 
+    emit_queries: Optional[bool] = pydantic.Field(
+        default=False,
+        description="Whether to emit queries aspects. Default is False.",
+    )
+
+    emit_assets: Optional[bool] = pydantic.Field(
+        default=True,
+        description="Whether to emit assets aspects. Default is True.",
+    )
+
     debug_mode: Optional[bool] = pydantic.Field(
         default=False,
         description="Whether to enable debug mode",
@@ -244,9 +267,10 @@ def job_url_generator(dagster_url: str, dagster_environment: DagsterEnvironment)
     return base_url
 
 
-class DagsterGenerator:
-    asset_group_name_cache: Dict[str, str] = {}
+DATAHUB_ASSET_GROUP_NAME_CACHE: Dict[str, str] = {}
+
 
+class DagsterGenerator:
     def __init__(
         self,
         logger: Logger,
@@ -299,17 +323,16 @@ def update_asset_group_name_cache(
             if asset_def:
                 for key, group_name in asset_def.group_names_by_key.items():
                     asset_urn = self.dataset_urn_from_asset(key.path)
-                    DagsterGenerator.asset_group_name_cache[
-                        asset_urn.urn()
-                    ] = group_name
+                    DATAHUB_ASSET_GROUP_NAME_CACHE[asset_urn.urn()] = group_name
                     if self.config.debug_mode:
                         self.logger.debug(
                             f"Asset group name cache updated: {asset_urn.urn()} -> {group_name}"
                         )
         if self.config.debug_mode:
             self.logger.debug(
-                f"Asset group name cache: {DagsterGenerator.asset_group_name_cache}"
+                f"Asset group name cache: {DATAHUB_ASSET_GROUP_NAME_CACHE}"
             )
+        self.logger.info(f"Asset group name cache: {DATAHUB_ASSET_GROUP_NAME_CACHE}")
 
     def path_metadata_resolver(self, value: PathMetadataValue) -> Optional[DatasetUrn]:
         """
@@ -348,6 +371,8 @@ def generate_dataflow(
         job_snapshot: JobSnapshot,
         env: str,
         platform_instance: Optional[str] = None,
+        remove_double_underscores: bool = True,
+        add_asset_group_tag: bool = True,
     ) -> DataFlow:
         """
         Generates a Dataflow object from an Dagster Job Snapshot
@@ -359,17 +384,34 @@ def generate_dataflow(
         if self.dagster_environment.is_cloud:
             id = f"{self.dagster_environment.branch}/{self.dagster_environment.module}/{job_snapshot.name}"
         else:
-            id = f"{self.dagster_environment.module}/{job_snapshot.name}"
+            module_name = (
+                self.dagster_environment.module
+                if self.dagster_environment.module
+                else self.dagster_environment.branch
+            )
+            id = f"{module_name}/{job_snapshot.name}"
+
+        flow_name = job_snapshot.name
+        if remove_double_underscores and flow_name.split("__"):
+            flow_name = flow_name.split("__")[-1]
 
         dataflow = DataFlow(
             orchestrator=Constant.ORCHESTRATOR,
             id=id,
             env=env,
-            name=job_snapshot.name,
+            name=flow_name,
             platform_instance=platform_instance,
         )
+
         dataflow.description = job_snapshot.description
         dataflow.tags = set(job_snapshot.tags.keys())
+        if add_asset_group_tag:
+            asset_group = self.get_asset_group_from_op_name(
+                job_snapshot.name.split("__")
+            )
+            if asset_group:
+                dataflow.tags.add(f"asset_group:{asset_group}")
+
         if self.config.dagster_url:
             dataflow.url = f"{job_url_generator(dagster_url=self.config.dagster_url, dagster_environment=self.dagster_environment)}/jobs/{job_snapshot.name}"
         flow_property_bag: Dict[str, str] = {}
@@ -387,6 +429,8 @@ def generate_datajob(
         input_datasets: Dict[str, Set[DatasetUrn]],
         output_datasets: Dict[str, Set[DatasetUrn]],
         platform_instance: Optional[str] = None,
+        remove_double_underscores: bool = True,
+        add_asset_group_tag: bool = True,
     ) -> DataJob:
         """
         Generates a Datajob object from an Dagster op snapshot
@@ -404,8 +448,13 @@ def generate_datajob(
             flow_id = f"{self.dagster_environment.branch}/{self.dagster_environment.module}/{job_snapshot.name}"
             job_id = f"{self.dagster_environment.branch}/{self.dagster_environment.module}/{op_def_snap.name}"
         else:
-            flow_id = f"{self.dagster_environment.module}/{job_snapshot.name}"
-            job_id = f"{self.dagster_environment.module}/{op_def_snap.name}"
+            module_name = (
+                self.dagster_environment.module
+                if self.dagster_environment.module
+                else self.dagster_environment.branch
+            )
+            flow_id = f"{module_name}/{job_snapshot.name}"
+            job_id = f"{module_name}/{op_def_snap.name}"
 
         dataflow_urn = DataFlowUrn.create_from_ids(
             orchestrator=Constant.ORCHESTRATOR,
@@ -413,10 +462,15 @@ def generate_datajob(
             env=env,
             platform_instance=platform_instance,
         )
+
+        job_name = op_def_snap.name
+        if remove_double_underscores and job_name.split("__"):
+            job_name = job_name.split("__")[-1]
+
         datajob = DataJob(
             id=job_id,
             flow_urn=dataflow_urn,
-            name=op_def_snap.name,
+            name=job_name,
         )
 
         if self.config.dagster_url:
@@ -425,6 +479,13 @@ def generate_datajob(
         datajob.description = op_def_snap.description
         datajob.tags = set(op_def_snap.tags.keys())
 
+        if add_asset_group_tag:
+            asset_group = self.get_asset_group_from_op_name(
+                op_def_snap.name.split("__")
+            )
+            if asset_group:
+                datajob.tags.add(f"asset_group:{asset_group}")
+
         inlets: Set[DatasetUrn] = set()
         # Add upstream dependencies for this op
         for upstream_op_name in step_deps[op_def_snap.name]:
@@ -461,7 +522,7 @@ def generate_datajob(
         # Also, add datahub inputs/outputs if present in input/output metatdata.
         for input_def_snap in op_def_snap.input_def_snaps:
             job_property_bag[f"input.{input_def_snap.name}"] = str(
-                input_def_snap._asdict()
+                input_def_snap.__dict__
             )
             if Constant.DATAHUB_INPUTS in input_def_snap.metadata:
                 datajob.inlets.extend(
@@ -472,7 +533,7 @@ def generate_datajob(
 
         for output_def_snap in op_def_snap.output_def_snaps:
             job_property_bag[f"output_{output_def_snap.name}"] = str(
-                output_def_snap._asdict()
+                output_def_snap.__dict__
             )
             if (
                 Constant.DATAHUB_OUTPUTS in output_def_snap.metadata
@@ -668,9 +729,9 @@ def generate_asset_group_tag(
         if not target_urn:
             target_urn = asset_urn
         self.logger.info(
-            f"Getting {asset_urn.urn()} from Asset Cache: {DagsterGenerator.asset_group_name_cache}"
+            f"Getting {asset_urn.urn()} from Asset Cache: {DATAHUB_ASSET_GROUP_NAME_CACHE}"
         )
-        group_name = DagsterGenerator.asset_group_name_cache.get(asset_urn.urn())
+        group_name = DATAHUB_ASSET_GROUP_NAME_CACHE.get(asset_urn.urn())
         if group_name:
             current_tags: Optional[GlobalTagsClass] = graph.get_aspect(
                 entity_urn=target_urn.urn(),
@@ -699,6 +760,20 @@ def generate_asset_group_tag(
 
         return None
 
+    def _gen_entity_browsepath_aspect(
+        self,
+        entity_urn: str,
+        paths: List[str],
+    ) -> MetadataWorkUnit:
+        entries = [BrowsePathEntryClass(id=path) for path in paths]
+        if self.config.platform_instance:
+            urn = make_dataplatform_instance_urn("asset", self.config.platform_instance)
+            entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries
+        return MetadataChangeProposalWrapper(
+            entityUrn=entity_urn,
+            aspect=BrowsePathsV2Class(entries),
+        ).as_workunit()
+
     def emit_asset(
         self,
         graph: DataHubGraph,
@@ -772,7 +847,8 @@ def emit_asset(
                             origin=DatasetUrn.create_from_string(downstream).env,
                         ),
                     )
-                    self.logger.info(f"mcp: {mcp}")
+                    if self.config.debug_mode:
+                        self.logger.info(f"mcp: {mcp}")
                     graph.emit_mcp(mcp)
 
                 patch_builder = DatasetPatchBuilder(downstream)
@@ -808,20 +884,38 @@ def emit_asset(
                 for patch_event in patch_builder.build():
                     graph.emit_mcp(patch_event)
 
-        self.logger.info(f"asset_key: {asset_key}")
+        if self.config.debug_mode:
+            self.logger.info(f"asset_key: {asset_key}")
         self.generate_browse_path(asset_key=asset_key, urn=dataset_urn, graph=graph)
         return dataset_urn
 
+    def get_asset_group_from_op_name(self, asset_key: Sequence[str]) -> Optional[str]:
+        """
+        Get asset group name from op name
+        """
+        asset_urn = self.dataset_urn_from_asset(asset_key).urn()
+        asset_group_name = DATAHUB_ASSET_GROUP_NAME_CACHE.get(asset_urn)
+        if asset_group_name:
+            self.logger.info(
+                f"asset_key: {asset_key}, urn: {asset_urn}, asset_group_name: {asset_group_name}"
+            )
+            return asset_group_name
+        else:
+            self.logger.info(
+                f"asset_key: {asset_key}, urn: {asset_urn} not in {DATAHUB_ASSET_GROUP_NAME_CACHE}, asset_group_name: None"
+            )
+
+        return None
+
     def generate_browse_path(
         self, asset_key: Sequence[str], urn: Urn, graph: DataHubGraph
     ) -> None:
         """
         Generate browse path from asset key
         """
-        asset_group_name = DagsterGenerator.asset_group_name_cache.get(
-            self.dataset_urn_from_asset(asset_key).urn()
-        )
         browsePaths: List[BrowsePathEntryClass] = []
+
+        asset_group_name = self.get_asset_group_from_op_name(asset_key)
         if asset_group_name:
             browsePaths.append(BrowsePathEntryClass(asset_group_name))
 
@@ -835,3 +929,51 @@ def generate_browse_path(
             ),
         )
         graph.emit_mcp(mcp)
+
+    def gen_query_aspect(
+        self,
+        graph: DataHubGraph,
+        platform: str,
+        query_subject_urns: List[str],
+        query: str,
+        job_urn: Optional[str] = None,
+    ) -> None:
+        """
+        Generate query aspect for lineage
+        """
+        query_id = get_query_fingerprint(query, platform)
+
+        aspects = [
+            QueryPropertiesClass(
+                statement=QueryStatementClass(
+                    value=query,
+                    language=QueryLanguageClass.SQL,
+                ),
+                source=QuerySourceClass.SYSTEM,
+                origin=job_urn if job_urn else None,
+                created=AuditStampClass(
+                    make_ts_millis(datetime.now(tz=timezone.utc)),
+                    actor=_DEFAULT_USER_URN.urn(),
+                ),
+                lastModified=AuditStampClass(
+                    make_ts_millis(datetime.now(tz=timezone.utc)),
+                    actor=_DEFAULT_USER_URN.urn(),
+                ),
+            ),
+            QuerySubjectsClass(
+                subjects=[QuerySubjectClass(entity=urn) for urn in query_subject_urns]
+            ),
+            DataPlatformInstanceClass(
+                platform=make_data_platform_urn(platform),
+            ),
+            SubTypesClass(
+                typeNames=["Query"],
+            ),
+        ]
+
+        mcps = MetadataChangeProposalWrapper.construct_many(
+            entityUrn=f"urn:li:query:dagster_{query_id}",
+            aspects=aspects,
+        )
+        for mcp in mcps:
+            graph.emit_mcp(mcp)
diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
index 4633014222d05..bccdb4ac7922a 100644
--- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
+++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
@@ -1,5 +1,6 @@
 import os
 import traceback
+import warnings
 from collections import defaultdict
 from types import ModuleType
 from typing import Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, Union
@@ -27,10 +28,15 @@
 from dagster._core.definitions.multi_asset_sensor_definition import (
     AssetMaterializationFunctionReturn,
 )
-from dagster._core.definitions.sensor_definition import (
-    DefaultSensorStatus,
-    RawSensorEvaluationFunctionReturn,
-)
+from dagster._core.definitions.sensor_definition import DefaultSensorStatus
+
+# This SensorReturnTypesUnion is from Dagster 1.9.1+ and is not available in older versions
+# of Dagster. We need to import it conditionally to avoid breaking compatibility with older
+try:
+    from dagster._core.definitions.sensor_definition import SensorReturnTypesUnion
+except ImportError:
+    from dagster._core.definitions.sensor_definition import RawSensorEvaluationFunctionReturn as SensorReturnTypesUnion  # type: ignore
+
 from dagster._core.definitions.target import ExecutableDefinition
 from dagster._core.definitions.unresolved_asset_job_definition import (
     UnresolvedAssetJobDefinition,
@@ -38,15 +44,18 @@
 from dagster._core.events import DagsterEventType, HandledOutputData, LoadedInputData
 from dagster._core.execution.stats import RunStepKeyStatsSnapshot
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
 from datahub.metadata.schema_classes import SubTypesClass
 from datahub.sql_parsing.sqlglot_lineage import (
     SqlParsingResult,
     create_lineage_sql_parsed_result,
 )
 from datahub.utilities.urns.dataset_urn import DatasetUrn
+from datahub.utilities.urns.error import InvalidUrnError
 
 from datahub_dagster_plugin.client.dagster_generator import (
+    DATAHUB_ASSET_GROUP_NAME_CACHE,
+    Constant,
     DagsterEnvironment,
     DagsterGenerator,
     DatahubDagsterSourceConfig,
@@ -182,7 +191,17 @@ def __init__(
         if config:
             self.config = config
         else:
-            self.config = DatahubDagsterSourceConfig()
+            # This is a temporary warning for backwards compatibility. Eventually, we'll remove this
+            # branch and make the config required.
+            warnings.warn(
+                "Using the default DataHub client config is deprecated. Pass in a config object explicitly.",
+                stacklevel=2,
+            )
+            self.config = DatahubDagsterSourceConfig(
+                datahub_client_config=DatahubClientConfig(
+                    server=Constant.DEFAULT_DATAHUB_REST_URL
+                )
+            )
         self.graph = DataHubGraph(
             self.config.datahub_client_config,
         )
@@ -256,7 +275,6 @@ def get_dagster_environment(
                 module = code_pointer.module
             else:
                 context.log.error("Unable to get Module")
-                return None
 
         dagster_environment = DagsterEnvironment(
             is_cloud=os.getenv("DAGSTER_CLOUD_IS_BRANCH_DEPLOYMENT", None) is not None,
@@ -314,100 +332,188 @@ def process_asset_materialization(
         dataset_inputs: Dict[str, Set[DatasetUrn]],
         dataset_outputs: Dict[str, Set[DatasetUrn]],
     ) -> None:
-        if (
-            log.dagster_event
-            and log.dagster_event.event_type == DagsterEventType.ASSET_MATERIALIZATION
-        ):
-            assert log.step_key
+        if not self._is_valid_asset_materialization(log):
+            return
 
-            materialization = log.asset_materialization
-            if not materialization:
-                return
+        asset_materialization = log.asset_materialization
+        if asset_materialization is None:
+            return
 
-            properties = {
-                key: str(value) for (key, value) in materialization.metadata.items()
-            }
-            asset_key = materialization.asset_key.path
+        asset_key = asset_materialization.asset_key.path
+        asset_downstream_urn = self._get_asset_downstream_urn(
+            log, context, dagster_generator, list(asset_key)
+        )
 
-            asset_downstream_urn: Optional[DatasetUrn] = None
-            # If DataHub Urn is set then we prefer that as downstream urn
-            if materialization.metadata.get("datahub_urn") and isinstance(
-                materialization.metadata.get("datahub_urn"), TextMetadataValue
-            ):
-                try:
-                    asset_downstream_urn = DatasetUrn.from_string(
-                        str(materialization.metadata["datahub_urn"].text)
-                    )
-                    context.log.info(
-                        f"asset_downstream_urn from metadata datahub_urn: {asset_downstream_urn}"
-                    )
-                except Exception as e:
-                    context.log.error(f"Error in parsing datahub_urn: {e}")
+        if not asset_downstream_urn:
+            return
 
-            if not asset_downstream_urn:
-                asset_downstream_urn = (
-                    dagster_generator.asset_keys_to_dataset_urn_converter(asset_key)
+        properties = {
+            key: str(value) for (key, value) in asset_materialization.metadata.items()
+        }
+        upstreams, downstreams = self._process_lineage(
+            context=context,
+            dagster_generator=dagster_generator,
+            log=log,
+            asset_downstream_urn=asset_downstream_urn,
+        )
+
+        self._emit_or_connect_asset(
+            context,
+            dagster_generator,
+            log,
+            list(asset_key),
+            properties,
+            upstreams,
+            downstreams,
+            dataset_inputs,
+            dataset_outputs,
+        )
+
+    def _is_valid_asset_materialization(self, log: EventLogEntry) -> bool:
+        return (
+            log.dagster_event is not None
+            and log.dagster_event.event_type == DagsterEventType.ASSET_MATERIALIZATION
+            and log.step_key is not None
+            and log.asset_materialization is not None
+        )
+
+    def _get_asset_downstream_urn(
+        self,
+        log: EventLogEntry,
+        context: RunStatusSensorContext,
+        dagster_generator: DagsterGenerator,
+        asset_key: List[str],
+    ) -> Optional[DatasetUrn]:
+        materialization = log.asset_materialization
+        if materialization is None:
+            return None
+
+        asset_downstream_urn: Optional[DatasetUrn] = None
+
+        if materialization.metadata.get("datahub_urn") and isinstance(
+            materialization.metadata.get("datahub_urn"), TextMetadataValue
+        ):
+            try:
+                asset_downstream_urn = DatasetUrn.from_string(
+                    str(materialization.metadata["datahub_urn"].text)
                 )
                 context.log.info(
-                    f"asset_downstream_urn from asset keys: {asset_downstream_urn}"
+                    f"asset_downstream_urn from metadata datahub_urn: {asset_downstream_urn}"
                 )
+            except Exception as e:
+                context.log.error(f"Error in parsing datahub_urn: {e}")
 
-            if asset_downstream_urn:
-                context.log.info(f"asset_downstream_urn: {asset_downstream_urn}")
+        if not asset_downstream_urn:
+            asset_downstream_urn = (
+                dagster_generator.asset_keys_to_dataset_urn_converter(asset_key)
+            )
+            context.log.info(
+                f"asset_downstream_urn from asset keys: {asset_downstream_urn}"
+            )
 
-                downstreams = {asset_downstream_urn.urn()}
-                context.log.info(f"downstreams: {downstreams}")
-                upstreams: Set[str] = set()
-                if self.config.enable_asset_query_metadata_parsing:
-                    try:
-                        if (
-                            materialization
-                            and materialization.metadata
-                            and materialization.metadata.get("Query")
-                            and isinstance(
-                                materialization.metadata.get("Query"), TextMetadataValue
-                            )
-                        ):
-                            query_metadata = materialization.metadata.get("Query")
-                            assert query_metadata
-                            lineage = self.parse_sql(
-                                context=context,
-                                sql_query=str(query_metadata.text),
-                                env=asset_downstream_urn.env,
-                                platform=asset_downstream_urn.platform.replace(
-                                    "urn:li:dataPlatform:", ""
-                                ),
+        return asset_downstream_urn
+
+    def _process_lineage(
+        self,
+        context: RunStatusSensorContext,
+        dagster_generator: DagsterGenerator,
+        log: EventLogEntry,
+        asset_downstream_urn: DatasetUrn,
+    ) -> Tuple[Set[str], Set[str]]:
+        downstreams = {asset_downstream_urn.urn()}
+        upstreams: Set[str] = set()
+
+        if (
+            log.asset_materialization
+            and self.config.enable_asset_query_metadata_parsing
+        ):
+            try:
+                query_metadata = log.asset_materialization.metadata.get("Query")
+                if isinstance(query_metadata, TextMetadataValue):
+                    lineage = self.parse_sql(
+                        context=context,
+                        sql_query=str(query_metadata.text),
+                        env=asset_downstream_urn.env,
+                        platform=asset_downstream_urn.platform.replace(
+                            "urn:li:dataPlatform:", ""
+                        ),
+                    )
+                    if lineage and lineage.downstreams:
+                        if self.config.emit_queries:
+                            dagster_generator.gen_query_aspect(
+                                graph=self.graph,
+                                platform=asset_downstream_urn.platform,
+                                query_subject_urns=lineage.upstreams
+                                + lineage.downstreams,
+                                query=str(query_metadata.text),
                             )
-                            # To make sure we don't process select queries check if downstream is present
-                            if lineage and lineage.downstreams:
-                                downstreams = downstreams.union(
-                                    set(lineage.downstreams)
-                                )
-                                upstreams = upstreams.union(set(lineage.upstreams))
-                                context.log.info(
-                                    f"Upstreams: {upstreams} Downstreams: {downstreams}"
-                                )
-                            else:
-                                context.log.info(
-                                    f"Lineage not found for {query_metadata.text}"
-                                )
-                        else:
-                            context.log.info("Query not found in metadata")
-                    except Exception as e:
-                        context.log.info(f"Error in processing asset logs: {e}")
-
-                # Emitting asset with upstreams and downstreams
-                dataset_urn = dagster_generator.emit_asset(
-                    self.graph,
-                    asset_key,
-                    materialization.description,
-                    properties,
-                    downstreams=downstreams,
-                    upstreams=upstreams,
-                    materialize_dependencies=self.config.materialize_dependencies,
+                        downstreams = downstreams.union(set(lineage.downstreams))
+                        upstreams = upstreams.union(set(lineage.upstreams))
+                        context.log.info(
+                            f"Upstreams: {upstreams} Downstreams: {downstreams}"
+                        )
+                    else:
+                        context.log.info(f"Lineage not found for {query_metadata.text}")
+                else:
+                    context.log.info("Query not found in metadata")
+            except Exception as e:
+                context.log.exception(f"Error in processing asset logs: {e}")
+
+        return upstreams, downstreams
+
+    def _emit_or_connect_asset(
+        self,
+        context: RunStatusSensorContext,
+        dagster_generator: DagsterGenerator,
+        log: EventLogEntry,
+        asset_key: List[str],
+        properties: Dict[str, str],
+        upstreams: Set[str],
+        downstreams: Set[str],
+        dataset_inputs: Dict[str, Set[DatasetUrn]],
+        dataset_outputs: Dict[str, Set[DatasetUrn]],
+    ) -> None:
+        if self.config.emit_assets:
+            context.log.info("Emitting asset metadata...")
+            dataset_urn = dagster_generator.emit_asset(
+                self.graph,
+                asset_key,
+                log.asset_materialization.description
+                if log.asset_materialization
+                else None,
+                properties,
+                downstreams=downstreams,
+                upstreams=upstreams,
+                materialize_dependencies=self.config.materialize_dependencies,
+            )
+            if log.step_key:
+                dataset_outputs[log.step_key].add(dataset_urn)
+        else:
+            context.log.info(
+                "Not emitting assets but connecting materialized dataset to DataJobs"
+            )
+            if log.step_key:
+                dataset_outputs[log.step_key] = dataset_outputs[log.step_key].union(
+                    [DatasetUrn.from_string(d) for d in downstreams]
                 )
 
-                dataset_outputs[log.step_key].add(dataset_urn)
+                dataset_upstreams: List[DatasetUrn] = []
+
+                for u in upstreams:
+                    try:
+                        dataset_upstreams.append(DatasetUrn.from_string(u))
+                    except InvalidUrnError as e:
+                        context.log.error(
+                            f"Error in parsing upstream dataset urn: {e}", exc_info=True
+                        )
+                        continue
+
+                dataset_inputs[log.step_key] = dataset_inputs[log.step_key].union(
+                    dataset_upstreams
+                )
+                context.log.info(
+                    f"Dataset Inputs: {dataset_inputs[log.step_key]} Dataset Outputs: {dataset_outputs[log.step_key]}"
+                )
 
     def process_asset_observation(
         self,
@@ -581,14 +687,14 @@ def _emit_asset_metadata(
             dagster_environment=dagster_environment,
         )
 
-        context.log.info("Emitting asset metadata...")
+        context.log.info(
+            f"Updating asset group name cache... {DATAHUB_ASSET_GROUP_NAME_CACHE}"
+        )
         dagster_generator.update_asset_group_name_cache(context)
 
         return SkipReason("Asset metadata processed")
 
-    def _emit_metadata(
-        self, context: RunStatusSensorContext
-    ) -> RawSensorEvaluationFunctionReturn:
+    def _emit_metadata(self, context: RunStatusSensorContext) -> SensorReturnTypesUnion:
         """
         Function to emit metadata for datahub rest.
         """
@@ -669,10 +775,21 @@ def _emit_metadata(
                 platform_instance=self.config.platform_instance,
             )
 
+            if (
+                dataflow.name
+                and dataflow.name.startswith("__ASSET_JOB")
+                and dataflow.name.split("__")
+            ):
+                dagster_generator.generate_browse_path(
+                    dataflow.name.split("__"), urn=dataflow.urn, graph=self.graph
+                )
+                dataflow.name = dataflow.name.split("__")[-1]
+
             dataflow.emit(self.graph)
             if self.config.debug_mode:
                 for mcp in dataflow.generate_mcp():
-                    context.log.debug(f"Emitted MCP: {mcp}")
+                    if self.config.debug_mode:
+                        context.log.debug(f"Emitted MCP: {mcp}")
 
             # Emit dagster job run which get mapped with datahub data process instance entity
             dagster_generator.emit_job_run(
@@ -709,6 +826,16 @@ def _emit_metadata(
                     input_datasets=dataset_inputs,
                 )
 
+                if (
+                    datajob.name
+                    and datajob.name.startswith("__ASSET_JOB")
+                    and datajob.name.split("__")
+                ):
+                    dagster_generator.generate_browse_path(
+                        datajob.name.split("__"), urn=datajob.urn, graph=self.graph
+                    )
+                    datajob.name = datajob.name.split("__")[-1]
+
                 datajob.emit(self.graph)
 
                 if self.config.debug_mode:
diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/conftest.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/conftest.py
new file mode 100644
index 0000000000000..5b9442d048672
--- /dev/null
+++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/conftest.py
@@ -0,0 +1,21 @@
+import pathlib
+import site
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from _pytest.config import Parser
+
+
+def pytest_addoption(parser: "Parser") -> None:
+    parser.addoption(
+        "--update-golden-files",
+        action="store_true",
+        default=False,
+    )
+
+
+# See https://coverage.readthedocs.io/en/latest/subprocess.html#configuring-python-for-sub-process-measurement
+coverage_startup_code = "import coverage; coverage.process_startup()"
+site_packages_dir = pathlib.Path(site.getsitepackages()[0])
+pth_file_path = site_packages_dir / "datahub_coverage_startup.pth"
+pth_file_path.write_text(coverage_startup_code)
diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/golden/golden_test_emit_metadata_mcps.json b/metadata-ingestion-modules/dagster-plugin/tests/unit/golden/golden_test_emit_metadata_mcps.json
new file mode 100644
index 0000000000000..afd67422d1564
--- /dev/null
+++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/golden/golden_test_emit_metadata_mcps.json
@@ -0,0 +1,658 @@
+[
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataFlowInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "etl",
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:dagster"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataFlowInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "etl",
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:dagster"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:0fa780ff92120ac3bf49d0bc070bbda2",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "job_snapshot_id": "28d03bf7b3138bea153a21f70f29e7c3ffa6ab0a",
+                "execution_plan_snapshot_id": "0f504b218cd28750ffa8d90a40e9647acba72021",
+                "has_repository_load_data": "False",
+                "tags": "{}",
+                "steps_succeeded": "2",
+                "steps_failed": "0",
+                "materializations": "0",
+                "expectations": "0",
+                "start_time": "1720681200.0",
+                "end_time": "1720681200.0"
+            },
+            "name": "12345678123456781234567812345678",
+            "type": "BATCH_SCHEDULED",
+            "created": {
+                "time": 1720681200000,
+                "actor": "urn:li:corpuser:datahub"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "output_result": "{'name': 'result', 'dagster_type_key': 'Any', 'description': None, 'is_required': True, 'metadata': {'datahub.outputs': JsonMetadataValue(data=['urn:li:dataset:(urn:li:dataPlatform:snowflake,tableB,PROD)'])}, 'is_dynamic': False}"
+            },
+            "name": "extract",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:0fa780ff92120ac3bf49d0bc070bbda2",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRelationships",
+    "aspect": {
+        "json": {
+            "parentTemplate": "urn:li:dataFlow:(dagster,prod/etl,PROD)",
+            "upstreamInstances": []
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:0fa780ff92120ac3bf49d0bc070bbda2",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1720681200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "STARTED"
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:0fa780ff92120ac3bf49d0bc070bbda2",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1720681200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "COMPLETE",
+            "result": {
+                "type": "SUCCESS",
+                "nativeResultType": "dagster"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [],
+            "outputDatasets": [],
+            "inputDatajobs": [],
+            "fineGrainedLineages": []
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:dagster"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "output_result": "{'name': 'result', 'dagster_type_key': 'Any', 'description': None, 'is_required': True, 'metadata': {'datahub.outputs': JsonMetadataValue(data=['urn:li:dataset:(urn:li:dataPlatform:snowflake,tableB,PROD)'])}, 'is_dynamic': False}"
+            },
+            "name": "extract",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [],
+            "outputDatasets": [],
+            "inputDatajobs": [],
+            "fineGrainedLineages": []
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:dagster"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "input.data": "{'name': 'data', 'dagster_type_key': 'Any', 'description': None, 'metadata': {'datahub.inputs': JsonMetadataValue(data=['urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)'])}}",
+                "output_result": "{'name': 'result', 'dagster_type_key': 'Any', 'description': None, 'is_required': True, 'metadata': {}, 'is_dynamic': False}"
+            },
+            "name": "transform",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)"
+            ],
+            "outputDatasets": [],
+            "inputDatajobs": [],
+            "fineGrainedLineages": []
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "name": "tableA",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:8e047f916b20c89e1874518890d6ff7e",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "step_key": "extract",
+                "attempts": "1",
+                "start_time": "1720681200.0",
+                "end_time": "1720681200.0"
+            },
+            "name": "12345678123456781234567812345678.prod/extract",
+            "type": "BATCH_SCHEDULED",
+            "created": {
+                "time": 1720681200000,
+                "actor": "urn:li:corpuser:datahub"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:dagster"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:8e047f916b20c89e1874518890d6ff7e",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRelationships",
+    "aspect": {
+        "json": {
+            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)",
+            "upstreamInstances": []
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:8e047f916b20c89e1874518890d6ff7e",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1720681200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "STARTED"
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:8e047f916b20c89e1874518890d6ff7e",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1720681200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "COMPLETE",
+            "result": {
+                "type": "SUCCESS",
+                "nativeResultType": "dagster"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "input.data": "{'name': 'data', 'dagster_type_key': 'Any', 'description': None, 'metadata': {'datahub.inputs': JsonMetadataValue(data=['urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)'])}}",
+                "output_result": "{'name': 'result', 'dagster_type_key': 'Any', 'description': None, 'is_required': True, 'metadata': {}, 'is_dynamic': False}"
+            },
+            "name": "transform",
+            "type": {
+                "string": "COMMAND"
+            },
+            "env": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)"
+            ],
+            "outputDatasets": [],
+            "inputDatajobs": [],
+            "fineGrainedLineages": []
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "name": "tableA",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:dagster"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "step_key": "transform",
+                "attempts": "1",
+                "start_time": "1720681200.0",
+                "end_time": "1720681200.0"
+            },
+            "name": "12345678123456781234567812345678.prod/transform",
+            "type": "BATCH_SCHEDULED",
+            "created": {
+                "time": 1720681200000,
+                "actor": "urn:li:corpuser:datahub"
+            }
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetKey",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "name": "tableA",
+            "origin": "PROD"
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRelationships",
+    "aspect": {
+        "json": {
+            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)",
+            "upstreamInstances": []
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceInput",
+    "aspect": {
+        "json": {
+            "inputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)"
+            ]
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1720681200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "STARTED"
+        }
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1720681200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "status": "COMPLETE",
+            "result": {
+                "type": "SUCCESS",
+                "nativeResultType": "dagster"
+            }
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py
index ac46cfd86fbb9..c951b959f85d4 100644
--- a/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py
+++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py
@@ -1,6 +1,11 @@
+import json
+import pathlib
+import tempfile
+import uuid
+from typing import Dict, List, Mapping, Sequence, Set
 from unittest.mock import Mock, patch
 
-import pytest
+import dagster._core.utils
 from dagster import (
     DagsterInstance,
     In,
@@ -11,12 +16,16 @@
     job,
     op,
 )
-from datahub.api.entities.dataprocess.dataprocess_instance import (
-    DataProcessInstanceKey,
-    InstanceRunResult,
+from dagster._core.definitions.job_definition import JobDefinition
+from dagster._core.definitions.repository_definition import (
+    RepositoryData,
+    RepositoryDefinition,
 )
-from datahub.configuration.source_common import DEFAULT_ENV
+from dagster._core.definitions.resource_definition import ResourceDefinition
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DatahubClientConfig
+from freezegun import freeze_time
+from utils.utils import PytestConfig, check_golden_file
 
 from datahub_dagster_plugin.client.dagster_generator import DatahubDagsterSourceConfig
 from datahub_dagster_plugin.sensors.datahub_sensors import (
@@ -24,12 +33,49 @@
     make_datahub_sensor,
 )
 
+FROZEN_TIME = "2024-07-11 07:00:00"
 
-@patch("datahub.ingestion.graph.client.DataHubGraph", autospec=True)
-@pytest.mark.skip(reason="disabling this test unti it will use proper golden files")
+call_num = 0
+
+
+def make_new_run_id_mock() -> str:
+    global call_num
+    call_num += 1
+    return f"test_run_id_{call_num}"
+
+
+dagster._core.utils.make_new_run_id = make_new_run_id_mock
+
+
+@patch("datahub_dagster_plugin.sensors.datahub_sensors.DataHubGraph", autospec=True)
 def test_datahub_sensor(mock_emit):
     instance = DagsterInstance.ephemeral()
-    context = build_sensor_context(instance=instance)
+
+    class DummyRepositoryData(RepositoryData):
+        def __init__(self):
+            self.sensors = []
+
+        def get_all_jobs(self) -> Sequence["JobDefinition"]:
+            return []
+
+        def get_top_level_resources(self) -> Mapping[str, "ResourceDefinition"]:
+            """Return all top-level resources in the repository as a list,
+            such as those provided to the Definitions constructor.
+
+            Returns:
+                List[ResourceDefinition]: All top-level resources in the repository.
+            """
+            return {}
+
+        def get_env_vars_by_top_level_resource(self) -> Mapping[str, Set[str]]:
+            return {}
+
+    repository_defintion = RepositoryDefinition(
+        name="testRepository", repository_data=DummyRepositoryData()
+    )
+    context = build_sensor_context(
+        instance=instance, repository_def=repository_defintion
+    )
     mock_emit.return_value = Mock()
 
     config = DatahubDagsterSourceConfig(
@@ -44,9 +90,13 @@ def test_datahub_sensor(mock_emit):
     assert isinstance(skip_reason, SkipReason)
 
 
-@patch("datahub_dagster_plugin.sensors.datahub_sensors.DatahubClient", autospec=True)
-@pytest.mark.skip(reason="disabling this test unti it will use proper golden files")
-def test_emit_metadata(mock_emit):
+TEST_UUIDS = ["uuid_{}".format(i) for i in range(10000)]
+
+
+@patch.object(uuid, "uuid4", side_effect=TEST_UUIDS)
+@patch("datahub_dagster_plugin.sensors.datahub_sensors.DataHubGraph", autospec=True)
+@freeze_time(FROZEN_TIME)
+def test_emit_metadata(mock_emit: Mock, pytestconfig: PytestConfig) -> None:
     mock_emitter = Mock()
     mock_emit.return_value = mock_emitter
 
@@ -87,7 +137,8 @@ def etl():
         transform(extract())
 
     instance = DagsterInstance.ephemeral()
-    result = etl.execute_in_process(instance=instance)
+    test_run_id = "12345678123456781234567812345678"
+    result = etl.execute_in_process(instance=instance, run_id=test_run_id)
 
     # retrieve the DagsterRun
     dagster_run = result.dagster_run
@@ -103,201 +154,25 @@ def etl():
         dagster_event=dagster_event,
     )
 
-    DatahubSensors()._emit_metadata(run_status_sensor_context)
-
-    expected_dataflow_urn = (
-        f"urn:li:dataFlow:(dagster,{dagster_run.job_name},{DEFAULT_ENV})"
-    )
-    assert mock_emitter.method_calls[1][1][0].aspectName == "dataFlowInfo"
-    assert mock_emitter.method_calls[1][1][0].entityUrn == expected_dataflow_urn
-    assert mock_emitter.method_calls[2][1][0].aspectName == "ownership"
-    assert mock_emitter.method_calls[2][1][0].entityUrn == expected_dataflow_urn
-    assert mock_emitter.method_calls[3][1][0].aspectName == "globalTags"
-    assert mock_emitter.method_calls[3][1][0].entityUrn == expected_dataflow_urn
-
-    dpi_id = DataProcessInstanceKey(
-        cluster=DEFAULT_ENV,
-        orchestrator="dagster",
-        id=dagster_run.run_id,
-    ).guid()
-    assert (
-        mock_emitter.method_calls[7][1][0].aspectName == "dataProcessInstanceProperties"
-    )
-    assert (
-        mock_emitter.method_calls[7][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[8][1][0].aspectName
-        == "dataProcessInstanceRelationships"
-    )
-    assert (
-        mock_emitter.method_calls[8][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[9][1][0].aspectName == "dataProcessInstanceRunEvent"
-    )
-    assert (
-        mock_emitter.method_calls[9][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[10][1][0].aspectName == "dataProcessInstanceRunEvent"
-    )
-    assert (
-        mock_emitter.method_calls[10][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[10][1][0].aspect.result.type
-        == InstanceRunResult.SUCCESS
-    )
-    assert mock_emitter.method_calls[11][1][0].aspectName == "dataJobInfo"
-    assert (
-        mock_emitter.method_calls[11][1][0].entityUrn
-        == f"urn:li:dataJob:({expected_dataflow_urn},extract)"
-    )
-    assert mock_emitter.method_calls[12][1][0].aspectName == "dataJobInputOutput"
-    assert (
-        mock_emitter.method_calls[12][1][0].entityUrn
-        == f"urn:li:dataJob:({expected_dataflow_urn},extract)"
-    )
-    assert mock_emitter.method_calls[13][1][0].aspectName == "status"
-    assert (
-        mock_emitter.method_calls[13][1][0].entityUrn
-        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableB,PROD)"
-    )
-    assert mock_emitter.method_calls[14][1][0].aspectName == "ownership"
-    assert (
-        mock_emitter.method_calls[14][1][0].entityUrn
-        == f"urn:li:dataJob:({expected_dataflow_urn},extract)"
-    )
-    assert mock_emitter.method_calls[15][1][0].aspectName == "globalTags"
-    assert (
-        mock_emitter.method_calls[15][1][0].entityUrn
-        == f"urn:li:dataJob:({expected_dataflow_urn},extract)"
-    )
-    dpi_id = DataProcessInstanceKey(
-        cluster=DEFAULT_ENV,
-        orchestrator="dagster",
-        id=f"{dagster_run.run_id}.extract",
-    ).guid()
-    assert (
-        mock_emitter.method_calls[21][1][0].aspectName
-        == "dataProcessInstanceProperties"
-    )
-    assert (
-        mock_emitter.method_calls[21][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[22][1][0].aspectName
-        == "dataProcessInstanceRelationships"
-    )
-    assert (
-        mock_emitter.method_calls[22][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert mock_emitter.method_calls[23][1][0].aspectName == "dataProcessInstanceOutput"
-    assert (
-        mock_emitter.method_calls[23][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert mock_emitter.method_calls[24][1][0].aspectName == "status"
-    assert (
-        mock_emitter.method_calls[24][1][0].entityUrn
-        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableB,PROD)"
-    )
-    assert (
-        mock_emitter.method_calls[25][1][0].aspectName == "dataProcessInstanceRunEvent"
-    )
-    assert (
-        mock_emitter.method_calls[25][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[26][1][0].aspectName == "dataProcessInstanceRunEvent"
-    )
-    assert (
-        mock_emitter.method_calls[26][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[26][1][0].aspect.result.type
-        == InstanceRunResult.SUCCESS
-    )
-    assert mock_emitter.method_calls[27][1][0].aspectName == "dataJobInfo"
-    assert (
-        mock_emitter.method_calls[27][1][0].entityUrn
-        == f"urn:li:dataJob:({expected_dataflow_urn},transform)"
-    )
-    assert mock_emitter.method_calls[28][1][0].aspectName == "dataJobInputOutput"
-    assert (
-        mock_emitter.method_calls[28][1][0].entityUrn
-        == f"urn:li:dataJob:({expected_dataflow_urn},transform)"
-    )
-    assert mock_emitter.method_calls[29][1][0].aspectName == "status"
-    assert (
-        mock_emitter.method_calls[29][1][0].entityUrn
-        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)"
-    )
-    assert mock_emitter.method_calls[30][1][0].aspectName == "ownership"
-    assert (
-        mock_emitter.method_calls[30][1][0].entityUrn
-        == f"urn:li:dataJob:({expected_dataflow_urn},transform)"
-    )
-    assert mock_emitter.method_calls[31][1][0].aspectName == "globalTags"
-    assert (
-        mock_emitter.method_calls[31][1][0].entityUrn
-        == f"urn:li:dataJob:({expected_dataflow_urn},transform)"
-    )
-    dpi_id = DataProcessInstanceKey(
-        cluster=DEFAULT_ENV,
-        orchestrator="dagster",
-        id=f"{dagster_run.run_id}.transform",
-    ).guid()
-    assert (
-        mock_emitter.method_calls[37][1][0].aspectName
-        == "dataProcessInstanceProperties"
-    )
-    assert (
-        mock_emitter.method_calls[37][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[38][1][0].aspectName
-        == "dataProcessInstanceRelationships"
-    )
-    assert (
-        mock_emitter.method_calls[38][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert mock_emitter.method_calls[39][1][0].aspectName == "dataProcessInstanceInput"
-    assert (
-        mock_emitter.method_calls[39][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert mock_emitter.method_calls[40][1][0].aspectName == "status"
-    assert (
-        mock_emitter.method_calls[40][1][0].entityUrn
-        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)"
-    )
-    assert (
-        mock_emitter.method_calls[41][1][0].aspectName == "dataProcessInstanceRunEvent"
-    )
-    assert (
-        mock_emitter.method_calls[41][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[42][1][0].aspectName == "dataProcessInstanceRunEvent"
-    )
-    assert (
-        mock_emitter.method_calls[42][1][0].entityUrn
-        == f"urn:li:dataProcessInstance:{dpi_id}"
-    )
-    assert (
-        mock_emitter.method_calls[42][1][0].aspect.result.type
-        == InstanceRunResult.SUCCESS
-    )
+    with tempfile.TemporaryDirectory() as tmp_path:
+        DatahubSensors()._emit_metadata(run_status_sensor_context)
+        mcpws: List[Dict] = []
+        for mock_call in mock_emitter.method_calls:
+            if not mock_call.args:
+                continue
+            mcpw = mock_call.args[0]
+            if isinstance(mcpw, MetadataChangeProposalWrapper):
+                mcpws.append(mcpw.to_obj(simplified_structure=True))
+
+        with open(f"{tmp_path}/test_emit_metadata_mcps.json", "w") as f:
+            json_object = json.dumps(mcpws, indent=2)
+            f.write(json_object)
+
+        check_golden_file(
+            pytestconfig=pytestconfig,
+            output_path=pathlib.Path(f"{tmp_path}/test_emit_metadata_mcps.json"),
+            golden_path=pathlib.Path(
+                "tests/unit/golden/golden_test_emit_metadata_mcps.json"
+            ),
+            ignore_paths=["root[*]['systemMetadata']['created']"],
+        )
diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/__init__.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/utils.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/utils.py
new file mode 100644
index 0000000000000..79e8b758bd0c0
--- /dev/null
+++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/utils.py
@@ -0,0 +1,30 @@
+import pathlib
+from typing import Sequence
+
+from datahub.testing.compare_metadata_json import assert_metadata_files_equal
+
+try:
+    from pytest import Config as PytestConfig  # type: ignore[attr-defined]
+except ImportError:
+    # Support for pytest 6.x.
+    from _pytest.config import Config as PytestConfig  # type: ignore
+
+__all__ = ["PytestConfig"]
+
+
+def check_golden_file(
+    pytestconfig: PytestConfig,
+    output_path: pathlib.Path,
+    golden_path: pathlib.Path,
+    ignore_paths: Sequence[str] = (),
+) -> None:
+    update_golden = pytestconfig.getoption("--update-golden-files")
+
+    assert_metadata_files_equal(
+        output_path=output_path,
+        golden_path=golden_path,
+        update_golden=update_golden,
+        copy_output=False,
+        ignore_paths=ignore_paths,
+        ignore_order=True,
+    )
diff --git a/metadata-ingestion-modules/gx-plugin/build.gradle b/metadata-ingestion-modules/gx-plugin/build.gradle
index 2288ae6bd8396..70bc3e0360a3a 100644
--- a/metadata-ingestion-modules/gx-plugin/build.gradle
+++ b/metadata-ingestion-modules/gx-plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'base'
 }
 
+apply from: "../../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -84,7 +86,7 @@ task testQuick(type: Exec, dependsOn: installDevTest) {
   outputs.dir("${venv_name}")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+    "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
 task buildWheel(type: Exec, dependsOn: [environmentSetup]) {
diff --git a/metadata-ingestion-modules/gx-plugin/setup.py b/metadata-ingestion-modules/gx-plugin/setup.py
index 404220ca4552e..40afc81a98f9c 100644
--- a/metadata-ingestion-modules/gx-plugin/setup.py
+++ b/metadata-ingestion-modules/gx-plugin/setup.py
@@ -15,15 +15,6 @@ def get_long_description():
 
 rest_common = {"requests", "requests_file"}
 
-# TODO: Can we move away from sqllineage and use sqlglot ??
-sqllineage_lib = {
-    "sqllineage==1.3.8",
-    # We don't have a direct dependency on sqlparse but it is a dependency of sqllineage.
-    # There have previously been issues from not pinning sqlparse, so it's best to pin it.
-    # Related: https://github.com/reata/sqllineage/issues/361 and https://github.com/reata/sqllineage/pull/360
-    "sqlparse==0.4.4",
-}
-
 _version: str = package_metadata["__version__"]
 _self_pin = (
     f"=={_version}"
@@ -38,13 +29,12 @@ def get_long_description():
     # GE added handling for higher version of jinja2 in version 0.15.12
     # https://github.com/great-expectations/great_expectations/pull/5382/files
     # TODO: support GX 0.18.0
-    "great-expectations>=0.15.12, <0.18.0",
+    "great-expectations>=0.15.12, <1.0.0",
     # datahub does not depend on traitlets directly but great expectations does.
     # https://github.com/ipython/traitlets/issues/741
     "traitlets<5.2.2",
     *rest_common,
-    *sqllineage_lib,
-    f"acryl-datahub[datahub-rest]{_self_pin}",
+    f"acryl-datahub[datahub-rest,sql-parser]{_self_pin}",
 }
 
 mypy_stubs = {
diff --git a/metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py b/metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py
index 76e43cf8c2c3d..2d89d26997d1f 100644
--- a/metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py
+++ b/metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py
@@ -8,6 +8,7 @@
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 import datahub.emitter.mce_builder as builder
+import packaging.version
 from datahub.cli.env_utils import get_boolean_env_variable
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.rest_emitter import DatahubRestEmitter
@@ -33,8 +34,9 @@
 )
 from datahub.metadata.com.linkedin.pegasus2avro.common import DataPlatformInstance
 from datahub.metadata.schema_classes import PartitionSpecClass, PartitionTypeClass
+from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
 from datahub.utilities._markupsafe_compat import MARKUPSAFE_PATCHED
-from datahub.utilities.sql_parser import DefaultSQLParser
+from datahub.utilities.urns.dataset_urn import DatasetUrn
 from great_expectations.checkpoint.actions import ValidationAction
 from great_expectations.core.batch import Batch
 from great_expectations.core.batch_spec import (
@@ -59,6 +61,16 @@
 from sqlalchemy.engine.base import Connection, Engine
 from sqlalchemy.engine.url import make_url
 
+# TODO: move this and version check used in tests to some common module
+try:
+    from great_expectations import __version__ as GX_VERSION  # type: ignore
+
+    has_name_positional_arg = packaging.version.parse(
+        GX_VERSION
+    ) >= packaging.version.Version("0.18.14")
+except Exception:
+    has_name_positional_arg = False
+
 if TYPE_CHECKING:
     from great_expectations.data_context.types.resource_identifiers import (
         GXCloudIdentifier,
@@ -78,6 +90,8 @@ class DataHubValidationAction(ValidationAction):
     def __init__(
         self,
         data_context: AbstractDataContext,
+        # this would capture `name` positional arg added in GX 0.18.14
+        *args: Union[str, Any],
         server_url: str,
         env: str = builder.DEFAULT_ENV,
         platform_alias: Optional[str] = None,
@@ -94,7 +108,12 @@ def __init__(
         name: str = "DataHubValidationAction",
     ):
 
-        super().__init__(data_context)
+        if has_name_positional_arg:
+            if len(args) >= 1 and isinstance(args[0], str):
+                name = args[0]
+            super().__init__(data_context, name)
+        else:
+            super().__init__(data_context)
         self.server_url = server_url
         self.env = env
         self.platform_alias = platform_alias
@@ -659,10 +678,23 @@ def get_dataset_partitions(self, batch_identifier, data_asset):
                     query=query,
                     customProperties=batchSpecProperties,
                 )
-                try:
-                    tables = DefaultSQLParser(query).get_tables()
-                except Exception as e:
-                    logger.warning(f"Sql parser failed on {query} with {e}")
+
+                data_platform = get_platform_from_sqlalchemy_uri(str(sqlalchemy_uri))
+                sql_parser_in_tables = create_lineage_sql_parsed_result(
+                    query=query,
+                    platform=data_platform,
+                    env=self.env,
+                    platform_instance=None,
+                    default_db=None,
+                )
+                tables = [
+                    DatasetUrn.from_string(table_urn).name
+                    for table_urn in sql_parser_in_tables.in_tables
+                ]
+                if sql_parser_in_tables.debug_info.table_error:
+                    logger.warning(
+                        f"Sql parser failed on {query} with {sql_parser_in_tables.debug_info.table_error}"
+                    )
                     tables = []
 
                 if len(set(tables)) != 1:
diff --git a/metadata-ingestion-modules/gx-plugin/tests/unit/test_great_expectations_action.py b/metadata-ingestion-modules/gx-plugin/tests/unit/test_great_expectations_action.py
index c870a4449abea..fcad17fa4b71d 100644
--- a/metadata-ingestion-modules/gx-plugin/tests/unit/test_great_expectations_action.py
+++ b/metadata-ingestion-modules/gx-plugin/tests/unit/test_great_expectations_action.py
@@ -30,7 +30,7 @@
 )
 from great_expectations.core.id_dict import IDDict
 from great_expectations.core.run_identifier import RunIdentifier
-from great_expectations.data_context import DataContext, FileDataContext
+from great_expectations.data_context import FileDataContext
 from great_expectations.data_context.types.resource_identifiers import (
     ExpectationSuiteIdentifier,
     ValidationResultIdentifier,
@@ -52,7 +52,7 @@
 
 
 @pytest.fixture(scope="function")
-def ge_data_context(tmp_path: str) -> DataContext:
+def ge_data_context(tmp_path: str) -> FileDataContext:
     return FileDataContext.create(tmp_path)
 
 
@@ -233,7 +233,7 @@ def ge_validation_result_suite_id_pandas() -> ValidationResultIdentifier:
 @mock.patch("datahub.emitter.rest_emitter.DatahubRestEmitter.emit_mcp", autospec=True)
 def test_DataHubValidationAction_sqlalchemy(
     mock_emitter: mock.MagicMock,
-    ge_data_context: DataContext,
+    ge_data_context: FileDataContext,
     ge_validator_sqlalchemy: Validator,
     ge_validation_result_suite: ExpectationSuiteValidationResult,
     ge_validation_result_suite_id: ValidationResultIdentifier,
@@ -337,7 +337,7 @@ def test_DataHubValidationAction_sqlalchemy(
 @mock.patch("datahub.emitter.rest_emitter.DatahubRestEmitter.emit_mcp", autospec=True)
 def test_DataHubValidationAction_pandas(
     mock_emitter: mock.MagicMock,
-    ge_data_context: DataContext,
+    ge_data_context: FileDataContext,
     ge_validator_pandas: Validator,
     ge_validation_result_suite_pandas: ExpectationSuiteValidationResult,
     ge_validation_result_suite_id_pandas: ValidationResultIdentifier,
@@ -399,7 +399,7 @@ def test_DataHubValidationAction_pandas(
 
 
 def test_DataHubValidationAction_graceful_failure(
-    ge_data_context: DataContext,
+    ge_data_context: FileDataContext,
     ge_validator_sqlalchemy: Validator,
     ge_validation_result_suite: ExpectationSuiteValidationResult,
     ge_validation_result_suite_id: ValidationResultIdentifier,
@@ -418,7 +418,7 @@ def test_DataHubValidationAction_graceful_failure(
 
 
 def test_DataHubValidationAction_not_supported(
-    ge_data_context: DataContext,
+    ge_data_context: FileDataContext,
     ge_validator_spark: Validator,
     ge_validation_result_suite: ExpectationSuiteValidationResult,
     ge_validation_result_suite_id: ValidationResultIdentifier,
diff --git a/metadata-ingestion-modules/prefect-plugin/build.gradle b/metadata-ingestion-modules/prefect-plugin/build.gradle
index bc091c6493364..a2b240cc8bf70 100644
--- a/metadata-ingestion-modules/prefect-plugin/build.gradle
+++ b/metadata-ingestion-modules/prefect-plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'base'
 }
 
+apply from: "../../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -82,14 +84,14 @@ task testQuick(type: Exec, dependsOn: installDevTest) {
   outputs.dir("${venv_name}")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s"
+    "pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s"
 }
 
 
 task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
+    "pytest -m 'not slow_integration' -vv ${get_coverage_args('full')} --continue-on-collection-errors --junit-xml=junit.full.xml"
 }
 
 
diff --git a/metadata-ingestion-modules/prefect-plugin/src/prefect_datahub/datahub_emitter.py b/metadata-ingestion-modules/prefect-plugin/src/prefect_datahub/datahub_emitter.py
index 5991503416aec..8617381cf1613 100644
--- a/metadata-ingestion-modules/prefect-plugin/src/prefect_datahub/datahub_emitter.py
+++ b/metadata-ingestion-modules/prefect-plugin/src/prefect_datahub/datahub_emitter.py
@@ -155,12 +155,11 @@ async def _get_flow_run_graph(self, flow_run_id: str) -> Optional[List[Dict]]:
             The flow run graph in json format.
         """
         try:
-            response = orchestration.get_client()._client.get(
+            response_coroutine = orchestration.get_client()._client.get(
                 f"/flow_runs/{flow_run_id}/graph"
             )
 
-            if asyncio.iscoroutine(response):
-                response = await response
+            response = await response_coroutine
 
             if hasattr(response, "json"):
                 response_json = response.json()
@@ -410,10 +409,9 @@ async def get_flow_run(flow_run_id: UUID) -> FlowRun:
             if not hasattr(client, "read_flow_run"):
                 raise ValueError("Client does not support async read_flow_run method")
 
-            response = client.read_flow_run(flow_run_id=flow_run_id)
+            response_coroutine = client.read_flow_run(flow_run_id=flow_run_id)
 
-            if asyncio.iscoroutine(response):
-                response = await response
+            response = await response_coroutine
 
             return FlowRun.parse_obj(response)
 
@@ -477,10 +475,9 @@ async def get_task_run(task_run_id: UUID) -> TaskRun:
             if not hasattr(client, "read_task_run"):
                 raise ValueError("Client does not support async read_task_run method")
 
-            response = client.read_task_run(task_run_id=task_run_id)
+            response_coroutine = client.read_task_run(task_run_id=task_run_id)
 
-            if asyncio.iscoroutine(response):
-                response = await response
+            response = await response_coroutine
 
             return TaskRun.parse_obj(response)
 
diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
index 4e3f1ca91766c..fc1409fbed74e 100644
--- a/metadata-ingestion/build.gradle
+++ b/metadata-ingestion/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'base'
 }
 
+apply from: "../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -11,10 +13,6 @@ if (!project.hasProperty("extra_pip_requirements")) {
     ext.extra_pip_requirements = ""
 }
 
-def get_coverage_arg(test_name) {
-  return "--cov-report xml:coverage_${test_name}.xml "
-}
-
 task checkPythonVersion(type: Exec) {
   commandLine python_executable, '-c',
     'import sys; sys.version_info >= (3, 8), f"Python version {sys.version_info[:2]} not allowed"'
@@ -25,20 +23,12 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
   inputs.file file('setup.py')
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
-    "${python_executable} -m venv ${venv_name} && " +
-    "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
-    "touch ${sentinel_file}"
-}
-
-task runPreFlightScript(type: Exec, dependsOn: environmentSetup) {
-  def sentinel_file = ".preflight_sentinel"
-  outputs.file(sentinel_file)
-  commandLine 'bash', '-c',
-    "scripts/datahub_preflight.sh && " +
+    "${python_executable} -m venv ${venv_name} && set -x && " +
+    "${venv_name}/bin/python -m pip install --upgrade uv && " +
     "touch ${sentinel_file}"
 }
 
-task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) {
+task installPackageOnly(type: Exec, dependsOn: environmentSetup) {
   def sentinel_file = "${venv_name}/.build_install_package_only_sentinel"
   inputs.file file('setup.py')
   outputs.file(sentinel_file)
@@ -142,7 +132,7 @@ task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJso
   inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
   inputs.files(project.fileTree(dir: "tests/"))
   outputs.dir("${venv_name}")
-  def cvg_arg = get_coverage_arg("quick")
+  def cvg_arg = get_coverage_args("quick")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "pytest ${cvg_arg} tests/unit --random-order --durations=20 -m 'not integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
@@ -174,19 +164,19 @@ task testSingle(dependsOn: [installDevTest]) {
 }
 
 task testIntegrationBatch0(type: Exec, dependsOn: [installDevTest]) {
-  def cvg_arg = get_coverage_arg("intBatch0")
+  def cvg_arg = get_coverage_args("intBatch0")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "pytest ${cvg_arg} --durations=50 -m 'integration_batch_0' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch0.xml"
 }
 task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) {
-  def cvg_arg = get_coverage_arg("intBatch1")
+  def cvg_arg = get_coverage_args("intBatch1")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "pytest ${cvg_arg} --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml"
 }
 task testIntegrationBatch2(type: Exec, dependsOn: [installDevTest]) {
-  def cvg_arg = get_coverage_arg("intBatch2")
+  def cvg_arg = get_coverage_args("intBatch2")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "pytest ${cvg_arg} --durations=20 -m 'integration_batch_2' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch2.xml"
diff --git a/metadata-ingestion/developing.md b/metadata-ingestion/developing.md
index 19a18c5275a3b..b713997d8286f 100644
--- a/metadata-ingestion/developing.md
+++ b/metadata-ingestion/developing.md
@@ -89,7 +89,16 @@ cd metadata-ingestion-modules/gx-plugin
 source venv/bin/activate
 datahub version  # should print "DataHub CLI version: unavailable (installed in develop mode)"
 ```
+### (Optional) Set up your Python environment for developing on Dagster Plugin
+
+From the repository root:
 
+```shell
+cd metadata-ingestion-modules/dagster-plugin
+../../gradlew :metadata-ingestion-modules:dagster-plugin:installDev
+source venv/bin/activate
+datahub version  # should print "DataHub CLI version: unavailable (installed in develop mode)"
+```
 ### Common setup issues
 
 Common issues (click to expand):
diff --git a/metadata-ingestion/docs/sources/athena/athena_recipe.yml b/metadata-ingestion/docs/sources/athena/athena_recipe.yml
index 540d8101737a3..c93047ffed9ff 100644
--- a/metadata-ingestion/docs/sources/athena/athena_recipe.yml
+++ b/metadata-ingestion/docs/sources/athena/athena_recipe.yml
@@ -1,6 +1,11 @@
 source:
   type: athena
   config:
+
+    # AWS Keys (Optional - Required only if local aws credentials are not set)
+    username: my_aws_access_key_id
+    password: my_aws_secret_access_key
+    
     # Coordinates
     aws_region: my_aws_region
     work_group: primary
diff --git a/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md b/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md
index 3d2a0509492bd..3433a853ea9b0 100644
--- a/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md
+++ b/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md
@@ -7,7 +7,7 @@ The business glossary source file should be a .yml file with the following top-l
 Example **Glossary**:
 
 ```yaml
-version: 1                                     			# the version of business glossary file config the config conforms to. Currently the only version released is `1`.
+version: "1"                                     			# the version of business glossary file config the config conforms to. Currently the only version released is `1`.
 source: DataHub                                			# the source format of the terms. Currently only supports `DataHub`
 owners:                                        			# owners contains two nested fields
   users:                                       		    # (optional) a list of user IDs
@@ -60,7 +60,7 @@ Example **GlossaryTerm**:
     - Shipping.CountryCode
     - Shipping.StreetAddress
   custom_properties:                                                        # (optional) a map of key/value pairs of arbitrary custom properties
-    - is_used_for_compliance_tracking: true
+    - is_used_for_compliance_tracking: "true"
   knowledge_links:                                                          # (optional) a list of **KnowledgeCard** related to this term. These appear as links on the glossary node's page
     - url: "https://en.wikipedia.org/wiki/Address"
       label: Wiki link
@@ -73,7 +73,7 @@ To see how these all work together, check out this comprehensive example busines
 <summary>Example business glossary file</summary>
 
 ```yaml
-version: 1
+version: "1"
 source: DataHub
 owners:
   users:
@@ -89,15 +89,15 @@ nodes:
       - name: Sensitive
         description: Sensitive Data
         custom_properties:
-          is_confidential: false
+          is_confidential: "false"
       - name: Confidential
         description: Confidential Data
         custom_properties:
-          is_confidential: true
+          is_confidential: "true"
       - name: HighlyConfidential
         description: Highly Confidential Data
         custom_properties:
-          is_confidential: true
+          is_confidential: "true"
         domain: Marketing
   - name: PersonalInformation
     description: All terms related to personal information
@@ -148,7 +148,7 @@ nodes:
         related_terms:
           - Housing.Kitchen.Cutlery
         custom_properties:
-          - is_used_for_compliance_tracking: true
+          - is_used_for_compliance_tracking: "true"
         knowledge_links:
           - url: "https://en.wikipedia.org/wiki/Address"
             label: Wiki link
@@ -237,7 +237,7 @@ Source file linked [here](https://github.com/datahub-project/datahub/blob/master
 
 ## Generating custom IDs for your terms
 
-IDs are normally inferred from the glossary term/node's name, see the `enable_auto_id` config. But, if you need a stable 
+IDs are normally inferred from the glossary term/node's name, see the `enable_auto_id` config. But, if you need a stable
 identifier, you can generate a custom ID for your term. It should be unique across the entire Glossary.
 
 Here's an example ID:
@@ -247,5 +247,5 @@ A note of caution: once you select a custom ID, it cannot be easily changed.
 
 ## Compatibility
 
-Compatible with version 1 of business glossary format. 
-The source will be evolved as we publish newer versions of this format.
\ No newline at end of file
+Compatible with version 1 of business glossary format.
+The source will be evolved as we publish newer versions of this format.
diff --git a/metadata-ingestion/docs/sources/cassandra/cassandra_pre.md b/metadata-ingestion/docs/sources/cassandra/cassandra_pre.md
new file mode 100644
index 0000000000000..095f4521d09ff
--- /dev/null
+++ b/metadata-ingestion/docs/sources/cassandra/cassandra_pre.md
@@ -0,0 +1,40 @@
+### Setup
+
+This integration pulls metadata directly from Cassandra databases, including both **DataStax Astra DB** and **Cassandra Enterprise Edition (EE)**. 
+
+You’ll need to have a Cassandra instance or an Astra DB setup with appropriate access permissions.
+
+#### Steps to Get the Required Information
+
+1. **Set Up User Credentials**:
+
+   - **For Astra DB**:
+     - Log in to your Astra DB Console.
+     - Navigate to **Organization Settings** > **Token Management**.
+     - Generate an **Application Token** with the required permissions for read access.
+     - Download the **Secure Connect Bundle** from the Astra DB Console.
+   - **For Cassandra EE**:
+     - Ensure you have a **username** and **password** with read access to the necessary keyspaces.
+
+2. **Permissions**:
+
+   - The user or token must have `SELECT` permissions that allow it to:
+     - Access metadata in system keyspaces (e.g., `system_schema`) to retrieve information about keyspaces, tables, columns, and views.
+     - Perform `SELECT` operations on the data tables if data profiling is enabled.
+
+3. **Verify Database Access**:
+   - For Astra DB: Ensure the **Secure Connect Bundle** is used and configured correctly.
+   - For Cassandra Opensource: Ensure the **contact point** and **port** are accessible.
+
+
+:::caution 
+
+When enabling profiling, make sure to set a limit on the number of rows to sample. Profiling large tables without a limit may lead to excessive resource consumption and slow performance.
+
+:::
+
+:::note
+
+For cloud configuration with Astra DB, it is necessary to specify the Secure Connect Bundle path in the configuration. For that reason, use the CLI to ingest metadata into DataHub.
+
+:::
diff --git a/metadata-ingestion/docs/sources/cassandra/cassandra_recipe.yml b/metadata-ingestion/docs/sources/cassandra/cassandra_recipe.yml
new file mode 100644
index 0000000000000..78cec5ef4f31c
--- /dev/null
+++ b/metadata-ingestion/docs/sources/cassandra/cassandra_recipe.yml
@@ -0,0 +1,30 @@
+source:
+  type: "cassandra"
+  config:
+    # Credentials for on prem cassandra
+    contact_point: "localhost"
+    port: 9042
+    username: "admin"
+    password: "password"
+
+    # Or
+    # Credentials Astra Cloud
+    #cloud_config:
+    #  secure_connect_bundle: "Path to Secure Connect Bundle (.zip)"
+    #  token: "Application Token"
+
+    # Optional Allow / Deny extraction of particular keyspaces.
+    keyspace_pattern:
+      allow: [".*"]
+
+    # Optional Allow / Deny extraction of particular tables.
+    table_pattern:
+      allow: [".*"]
+
+    # Optional 
+    profiling:
+      enabled: true
+      profile_table_level_only: true
+
+sink:
+  # config sinks
diff --git a/metadata-ingestion/docs/sources/dbt/dbt-cloud_pre.md b/metadata-ingestion/docs/sources/dbt/dbt-cloud_pre.md
index ac7283d21015a..9fb0d9f334208 100644
--- a/metadata-ingestion/docs/sources/dbt/dbt-cloud_pre.md
+++ b/metadata-ingestion/docs/sources/dbt/dbt-cloud_pre.md
@@ -2,9 +2,10 @@
 
 This source pulls dbt metadata directly from the dbt Cloud APIs.
 
-You'll need to have a dbt Cloud job set up to run your dbt project, and "Generate docs on run" should be enabled.
+Create a [service account token](https://docs.getdbt.com/docs/dbt-cloud-apis/service-tokens) with the "Metadata Only" permission.
+This is a read-only permission.
 
-The token should have the "read metadata" permission.
+You'll need to have a dbt Cloud job set up to run your dbt project, and "Generate docs on run" should be enabled.
 
 To get the required IDs, go to the job details page (this is the one with the "Run History" table), and look at the URL.
 It should look something like this: https://cloud.getdbt.com/next/deploy/107298/projects/175705/jobs/148094.
diff --git a/metadata-ingestion/docs/sources/dremio/README.md b/metadata-ingestion/docs/sources/dremio/README.md
new file mode 100644
index 0000000000000..7f191a56cb2dc
--- /dev/null
+++ b/metadata-ingestion/docs/sources/dremio/README.md
@@ -0,0 +1,11 @@
+### Concept Mapping
+
+Here's a table for **Concept Mapping** between Dremio and DataHub to provide a clear overview of how entities and concepts in Dremio are mapped to corresponding entities in DataHub:
+
+| Source Concept             | DataHub Concept | Notes                                                      |
+| -------------------------- | --------------- | ---------------------------------------------------------- |
+| **Physical Dataset/Table** | `Dataset`       | Subtype: `Table`                                           |
+| **Virtual Dataset/Views**  | `Dataset`       | Subtype: `View`                                            |
+| **Spaces**                 | `Container`     | Mapped to DataHub’s `Container` aspect. Subtype: `Space`   |
+| **Folders**                | `Container`     | Mapped as a `Container` in DataHub. Subtype: `Folder`      |
+| **Sources**                | `Container`     | Represented as a `Container` in DataHub. Subtype: `Source` |
diff --git a/metadata-ingestion/docs/sources/dremio/dremio.md b/metadata-ingestion/docs/sources/dremio/dremio.md
new file mode 100644
index 0000000000000..4a14f171b1e22
--- /dev/null
+++ b/metadata-ingestion/docs/sources/dremio/dremio.md
@@ -0,0 +1,29 @@
+### Starter Receipe for Dremio Cloud Instance 
+
+```
+source:
+  type: dremio
+  config:
+    # Authentication details
+    authentication_method: PAT        # Use Personal Access Token for authentication
+    password: <your_api_token>        # Replace <your_api_token> with your Dremio Cloud API token
+    is_dremio_cloud: True             # Set to True for Dremio Cloud instances
+    dremio_cloud_project_id: <project_id>  # Provide the Project ID for Dremio Cloud
+
+    # Enable query lineage tracking
+    include_query_lineage: True       
+
+    #Optional
+    source_mappings:
+      - platform: s3                 
+        source_name: samples       
+    
+    # Optional
+    schema_pattern:
+      allow:
+        - "<source_name>.<table_name>" 
+
+sink:
+    # Define your sink configuration here
+
+```
\ No newline at end of file
diff --git a/metadata-ingestion/docs/sources/dremio/dremio_pre.md b/metadata-ingestion/docs/sources/dremio/dremio_pre.md
new file mode 100644
index 0000000000000..6e2cd7065209f
--- /dev/null
+++ b/metadata-ingestion/docs/sources/dremio/dremio_pre.md
@@ -0,0 +1,25 @@
+### Setup
+
+This integration pulls metadata directly from the Dremio APIs.
+
+You'll need to have a Dremio instance up and running with access to the necessary datasets, and API access should be enabled with a valid token.
+
+The API token should have the necessary permissions to **read metadata** and **retrieve lineage**.
+
+#### Steps to Get the Required Information
+
+1. **Generate an API Token**:
+
+   - Log in to your Dremio instance.
+   - Navigate to your user profile in the top-right corner.
+   - Select **Generate API Token** to create an API token for programmatic access.
+
+2. **Permissions**:
+
+   - The token should have **read-only** or **admin** permissions that allow it to:
+     - View all datasets (physical and virtual).
+     - Access all spaces, folders, and sources.
+     - Retrieve dataset and column-level lineage information.
+
+3. **Verify External Data Source Permissions**:
+   - If Dremio is connected to external data sources (e.g., AWS S3, relational databases), ensure that Dremio has access to the credentials required for querying those sources.
diff --git a/metadata-ingestion/docs/sources/dremio/dremio_recipe.yml b/metadata-ingestion/docs/sources/dremio/dremio_recipe.yml
new file mode 100644
index 0000000000000..d18d19da2de84
--- /dev/null
+++ b/metadata-ingestion/docs/sources/dremio/dremio_recipe.yml
@@ -0,0 +1,36 @@
+source:
+  type: dremio
+  config:
+    # Coordinates
+    hostname: localhost
+    port: 9047
+    tls: true
+
+    # Credentials with personal access token(recommended)
+    authentication_method: PAT
+    password: pass
+    # OR Credentials with basic auth
+    # authentication_method: password
+    # username: user
+    # password: pass
+
+    #For cloud instance
+    #is_dremio_cloud: True
+    #dremio_cloud_project_id: <project_id>
+
+    include_query_lineage: True
+
+    ingest_owner: true
+
+    #Optional
+    source_mappings:
+      - platform: s3
+        source_name: samples
+
+    #Optional
+    schema_pattern:
+      allow:
+        - "<source_name>.<table_name>"
+
+sink:
+  # sink configs
\ No newline at end of file
diff --git a/metadata-ingestion/docs/sources/fivetran/fivetran_recipe.yml b/metadata-ingestion/docs/sources/fivetran/fivetran_recipe.yml
index 3495b89a7e193..af4d5c5792f41 100644
--- a/metadata-ingestion/docs/sources/fivetran/fivetran_recipe.yml
+++ b/metadata-ingestion/docs/sources/fivetran/fivetran_recipe.yml
@@ -25,7 +25,7 @@ source:
           client_email: "client_email"
           client_id: "client_id"
           private_key: "private_key"
-        dataset: "fivetran_log_dataset"	
+        dataset: "fivetran_log_dataset"
 
     # Optional - filter for certain connector names instead of ingesting everything.
     # connector_patterns:
@@ -35,7 +35,7 @@ source:
     # Optional -- A mapping of the connector's all sources to its database.
     # sources_to_database:
     #   connector_id: source_db
-    
+
     # Optional -- This mapping is optional and only required to configure platform-instance for source
     # A mapping of Fivetran connector id to data platform instance
     # sources_to_platform_instance:
diff --git a/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml b/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml
index 21734cd4e03fa..05e5205f7da41 100644
--- a/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml
+++ b/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml
@@ -22,3 +22,14 @@ source:
     soft_deleted_entities_cleanup:
       # Delete soft deleted entities which were deleted 10 days ago
       retention_days: 10
+    execution_request_cleanup:
+      # Minimum number of execution requests to keep, per ingestion source
+      keep_history_min_count: 10
+      # Maximum number of execution requests to keep, per ingestion source
+      keep_history_max_count: 1000
+      # Maximum number of days to keep execution requests for, per ingestion source
+      keep_history_max_days: 30
+      # Number of records per read operation
+      batch_read_size: 100
+      # Global switch for this cleanup task 
+      enabled: true
diff --git a/metadata-ingestion/docs/sources/iceberg/iceberg.md b/metadata-ingestion/docs/sources/iceberg/iceberg.md
index 7e40315a2e319..92aac5ffa6ce5 100644
--- a/metadata-ingestion/docs/sources/iceberg/iceberg.md
+++ b/metadata-ingestion/docs/sources/iceberg/iceberg.md
@@ -18,6 +18,8 @@ This ingestion source maps the following Source System Concepts to DataHub Conce
 
 ## Troubleshooting
 
-### [Common Issue]
+### Exceptions while increasing `processing_threads`
 
-[Provide description of common issues with this integration and steps to resolve]
+Each processing thread will open several files/sockets to download manifest files from blob storage. If you experience
+exceptions appearing when increasing `processing_threads` configuration parameter, try to increase limit of open
+files (i.e. using `ulimit` in Linux).
diff --git a/metadata-ingestion/docs/sources/kafka/kafka.md b/metadata-ingestion/docs/sources/kafka/kafka.md
index f9b50c13f0277..525f985be4662 100644
--- a/metadata-ingestion/docs/sources/kafka/kafka.md
+++ b/metadata-ingestion/docs/sources/kafka/kafka.md
@@ -102,7 +102,29 @@ source:
     connection:
       bootstrap: "broker:9092"
       schema_registry_url: http://localhost:8081
+```
+
+### OAuth Callback 
+The OAuth callback function can be set up using `config.connection.consumer_config.oauth_cb`.
+
+You need to specify a Python function reference in the format &lt;python-module&gt;:&lt;function-name&gt;.
+
+For example, in the configuration `oauth:create_token`, `create_token` is a function defined in `oauth.py`, and `oauth.py` must be accessible in the PYTHONPATH.
 
+```YAML
+source:
+  type: "kafka"
+  config:
+    # Set the custom schema registry implementation class
+    schema_registry_class: "datahub.ingestion.source.confluent_schema_registry.ConfluentSchemaRegistry"
+    # Coordinates
+    connection:
+      bootstrap: "broker:9092"
+      schema_registry_url: http://localhost:8081
+      consumer_config:
+        security.protocol: "SASL_PLAINTEXT"
+        sasl.mechanism: "OAUTHBEARER"
+        oauth_cb: "oauth:create_token"
 # sink configs
 ```
 
diff --git a/metadata-ingestion/docs/sources/neo4j/neo4j.md b/metadata-ingestion/docs/sources/neo4j/neo4j.md
new file mode 100644
index 0000000000000..d4dab2c6c7e1f
--- /dev/null
+++ b/metadata-ingestion/docs/sources/neo4j/neo4j.md
@@ -0,0 +1,20 @@
+## Integration Details
+
+<!-- Plain-language description of what this integration is meant to do.  -->
+<!-- Include details about where metadata is extracted from (ie. logs, source API, manifest, etc.)   -->
+Neo4j metadata will be ingested into DataHub using 
+`CALL apoc.meta.schema() YIELD value UNWIND keys(value) AS key RETURN key, value[key] AS value;`  
+The data that is returned will be parsed 
+and will be displayed as Nodes and Relationships in DataHub.  Each object will be tagged with describing what kind of DataHub
+object it is.  The defaults are 'Node' and 'Relationship'.  These tag values can be overwritten in the recipe.
+
+
+
+## Metadata Ingestion Quickstart
+
+### Prerequisites
+
+In order to ingest metadata from Neo4j, you will need:
+
+* Neo4j instance with APOC installed
+
diff --git a/metadata-ingestion/docs/sources/neo4j/neo4j_recipe.yml b/metadata-ingestion/docs/sources/neo4j/neo4j_recipe.yml
new file mode 100644
index 0000000000000..463d65e7ba323
--- /dev/null
+++ b/metadata-ingestion/docs/sources/neo4j/neo4j_recipe.yml
@@ -0,0 +1,12 @@
+source:
+    type: 'neo4j'
+    config:
+        uri: 'neo4j+ssc://host:7687'
+        username: 'neo4j'
+        password: 'password'
+        env: 'PROD'
+
+sink:
+  type: "datahub-rest"
+  config:
+    server: 'http://localhost:8080'
\ No newline at end of file
diff --git a/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md b/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md
index b581e5fc8f70d..1561a36d04c0c 100644
--- a/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md
+++ b/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md
@@ -10,17 +10,20 @@
 
 ## Concept mapping 
 
-| PowerBI              | Datahub                 |                                                                                               
-|-----------------------|---------------------|
-| `Dashboard`           | `Dashboard`         |
-| `Dataset's Table`     | `Dataset`           |
-| `Tile`                | `Chart`             |
-| `Report.webUrl`       | `Chart.externalUrl` |
-| `Workspace`           | `Container`         |
-| `Report`              | `Dashboard`         |
-| `Page`                | `Chart`             |
-
-If Tile is created from report then Chart.externalUrl is set to Report.webUrl.
+| PowerBI           | Datahub             |                                                                                               
+|-------------------|---------------------|
+| `Dashboard`       | `Dashboard`         |
+| `Dataset's Table` | `Dataset`           |
+| `Tile`            | `Chart`             |
+| `Report.webUrl`   | `Chart.externalUrl` |
+| `Workspace`       | `Container`         |
+| `Report`          | `Dashboard`         |
+| `PaginatedReport` | `Dashboard`         |
+| `Page`            | `Chart`             |
+| `App`             | `Dashboard`         |
+
+- If `Tile` is created from report then `Chart.externalUrl` is set to Report.webUrl.
+- The `Page` is unavailable for PowerBI PaginatedReport.
 
 ## Lineage
 
@@ -100,7 +103,7 @@ combine_result
 
 `Pattern-2` is *not* supported for upstream table lineage extraction as it uses nested item-selector i.e. {Source{[Schema="public",Item="book"]}[Data], Source{[Schema="public",Item="issue_history"]}[Data]} as argument to M-QUery table function i.e. Table.Combine
 
-`Pattern-1` is supported as it first assign the table from schema to variable and then variable is used in M-Query Table function i.e. Table.Combine
+`Pattern-1` is supported as it first assigns the table from schema to variable and then variable is used in M-Query Table function i.e. Table.Combine
 
 ## Extract endorsements to tags
 
@@ -110,18 +113,20 @@ Please note that the default implementation overwrites tags for the ingested ent
 
 ## Profiling
 
-The profiling implementation is done through querying [DAX query endpoint](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/execute-queries). Therefore the principal needs to have permission to query the datasets to be profiled. Usually this means that the service principal should have `Contributor` role for the workspace to be ingested. Profiling is done with column based queries to be able to handle wide datasets without timeouts.
+The profiling implementation is done through querying [DAX query endpoint](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/execute-queries). Therefore, the principal needs to have permission to query the datasets to be profiled. Usually this means that the service principal should have `Contributor` role for the workspace to be ingested. Profiling is done with column-based queries to be able to handle wide datasets without timeouts.
 
-Take into account that the profiling implementation exeutes fairly big amount of DAX queries and for big datasets this is substantial load to the PowerBI system.
+Take into account that the profiling implementation executes a fairly big number of DAX queries, and for big datasets this is a significant load to the PowerBI system.
 
-The `profiling_pattern` setting may be used to limit profiling actions to only a certain set of resources in PowerBI. Both allow and deny rules are matched against following pattern for every table in a PowerBI Dataset: `workspace_name.dataset_name.table_name`. User may limit profiling with these settings at table level, dataset level or workspace level.
+The `profiling_pattern` setting may be used to limit profiling actions to only a certain set of resources in PowerBI. Both allowed and deny rules are matched against the following pattern for every table in a PowerBI Dataset: `workspace_name.dataset_name.table_name`. Users may limit profiling with these settings at table level, dataset level or workspace level.
 
 ## Admin Ingestion vs. Basic Ingestion
 PowerBI provides two sets of API i.e. [Basic API and Admin API](https://learn.microsoft.com/en-us/rest/api/power-bi/). 
 
-The Basic API returns metadata of PowerBI resources where service principal has granted access explicitly on resources whereas Admin API returns metadata of all PowerBI resources irrespective of whether service principal has granted or doesn't granted access explicitly  on resources.
+The Basic API returns metadata of PowerBI resources where service principal has granted access explicitly on resources,
+whereas Admin API returns metadata of all PowerBI resources irrespective of whether service principal has granted
+or doesn't grant access explicitly on resources.
 
-The Admin Ingestion (explain below) is the recommended way to execute PowerBI ingestion as this ingestion can extract most of the metadata.
+The Admin Ingestion (explained below) is the recommended way to execute PowerBI ingestion as this ingestion can extract most of the metadata.
 
 
 ### Admin Ingestion: Service Principal As Admin in Tenant Setting and Added as Member In Workspace
@@ -140,8 +145,9 @@ PowerBI Source would be able to ingest below listed metadata of that particular
   - Endorsement as tag
   - Dashboards 
   - Reports 
-  - Dashboard's Tiles
-  - Report's Pages
+  - Dashboard Tiles
+  - Report Pages
+  - App
 
 If you don't want to add a service principal as a member in your workspace, then you can enable the `admin_apis_only: true` in recipe to use PowerBI Admin API only. 
 
@@ -152,7 +158,7 @@ Caveats of setting `admin_apis_only` to `true`:
 
 
 ### Basic Ingestion: Service Principal As Member In Workspace 
-If you have added service principal as `member` in workspace then PowerBI Source would be able ingest below metadata of that particular workspace 
+If you have added service principal as `member` in workspace then PowerBI Source would be able to ingest below metadata of that particular workspace 
 
   - Dashboards 
   - Reports 
diff --git a/metadata-ingestion/docs/sources/redash/redash.md b/metadata-ingestion/docs/sources/redash/redash.md
index 8f8c5c85496a0..f23a523cebc91 100644
--- a/metadata-ingestion/docs/sources/redash/redash.md
+++ b/metadata-ingestion/docs/sources/redash/redash.md
@@ -1,5 +1,2 @@
-Note! The integration can use an SQL parser to try to parse the tables the chart depends on. This parsing is disabled by default, 
-but can be enabled by setting `parse_table_names_from_sql: true`.  The default parser is based on the [`sqllineage`](https://pypi.org/project/sqllineage/) package.
-As this package doesn't officially support all the SQL dialects that Redash supports, the result might not be correct. You can, however, implement a
-custom parser and take it into use by setting the `sql_parser` configuration value. A custom SQL parser must inherit from `datahub.utilities.sql_parser.SQLParser`
-and must be made available to Datahub by ,for example, installing it. The configuration then needs to be set to `module_name.ClassName` of the parser.
+Note! The integration can use an SQL parser to try to parse the tables the chart depends on. This parsing is disabled by default,
+but can be enabled by setting `parse_table_names_from_sql: true`. The parser is based on the [`sqlglot`](https://pypi.org/project/sqlglot/) package.
diff --git a/metadata-ingestion/docs/sources/sac/sac_pre.md b/metadata-ingestion/docs/sources/sac/sac_pre.md
index c62cd81fa2753..624eb61f716f9 100644
--- a/metadata-ingestion/docs/sources/sac/sac_pre.md
+++ b/metadata-ingestion/docs/sources/sac/sac_pre.md
@@ -4,6 +4,7 @@
 
     - Purpose: API Access
     - Access:
+        - Story Listing
         - Data Import Service
     - Authorization Grant: Client Credentials
 
diff --git a/metadata-ingestion/docs/sources/sigma/sigma_pre.md b/metadata-ingestion/docs/sources/sigma/sigma_pre.md
index 382a2fe67b944..433f85a69f907 100644
--- a/metadata-ingestion/docs/sources/sigma/sigma_pre.md
+++ b/metadata-ingestion/docs/sources/sigma/sigma_pre.md
@@ -16,7 +16,7 @@ This source extracts the following:
 | Sigma                  | Datahub												         | Notes                            |
 |------------------------|---------------------------------------------------------------|----------------------------------|
 | `Workspace`            | [Container](../../metamodel/entities/container.md)     	     | SubType `"Sigma Workspace"`      |
-| `Workbook`             | [Container](../../metamodel/entities/container.md)            | SubType `"Sigma Workbook"`       |
+| `Workbook`             | [Dashboard](../../metamodel/entities/dashboard.md)            | SubType `"Sigma Workbook"`       |
 | `Page`                 | [Dashboard](../../metamodel/entities/dashboard.md)            |                                  |
 | `Element`              | [Chart](../../metamodel/entities/chart.md)                    |                                  |
 | `Dataset`              | [Dataset](../../metamodel/entities/dataset.md)                | SubType `"Sigma Dataset"`        |
diff --git a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md
index 75bd579417a48..4cfbc470e8c23 100644
--- a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md
+++ b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md
@@ -23,12 +23,16 @@ grant references on all external tables in database "<your-database>" to role da
 grant references on future external tables in database "<your-database>" to role datahub_role;
 grant references on all views in database "<your-database>" to role datahub_role;
 grant references on future views in database "<your-database>" to role datahub_role;
+grant monitor on all dynamic tables in database "<your-database>" to role datahub_role;
+grant monitor on future dynamic tables in database "<your-database>" to role datahub_role;
 
 // If you ARE using Snowflake Profiling or Classification feature: Grant select privileges to your tables
 grant select on all tables in database "<your-database>" to role datahub_role;
 grant select on future tables in database "<your-database>" to role datahub_role;
 grant select on all external tables in database "<your-database>" to role datahub_role;
 grant select on future external tables in database "<your-database>" to role datahub_role;
+grant select on all dynamic tables in database "<your-database>" to role datahub_role;
+grant select on future dynamic tables in database "<your-database>" to role datahub_role;
 
 // Create a new DataHub user and assign the DataHub role to it
 create user datahub_user display_name = 'DataHub' password='' default_role = datahub_role default_warehouse = '<your-warehouse>';
diff --git a/metadata-ingestion/docs/transformer/dataset_transformer.md b/metadata-ingestion/docs/transformer/dataset_transformer.md
index d48c6d2c1ab5b..66274ce64a8d2 100644
--- a/metadata-ingestion/docs/transformer/dataset_transformer.md
+++ b/metadata-ingestion/docs/transformer/dataset_transformer.md
@@ -122,12 +122,13 @@ transformers:
 ```
 ## Simple Add Dataset ownership 
 ### Config Details
-| Field              | Required | Type         | Default     | Description                                                         |
-|--------------------|----------|--------------|-------------|---------------------------------------------------------------------|
-| `owner_urns`       | ✅        | list[string] |             | List of owner urns.                                                 |
-| `ownership_type`   |          | string       | "DATAOWNER" | ownership type of the owners (either as enum or ownership type urn) |
-| `replace_existing` |          | boolean      | `false`     | Whether to remove ownership from entity sent by ingestion source.      |
-| `semantics`        |          | enum         | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS.    |
+| Field              | Required | Type         | Default     | Description                                                                                                |
+|--------------------|----------|--------------|-------------|------------------------------------------------------------------------------------------------------------|
+| `owner_urns`       | ✅        | list[string] |             | List of owner urns.                                                                                        |
+| `ownership_type`   |          | string       | "DATAOWNER" | ownership type of the owners (either as enum or ownership type urn)                                        |
+| `replace_existing` |          | boolean      | `false`     | Whether to remove ownership from entity sent by ingestion source.                                          |
+| `semantics`        |          | enum         | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS.                                           |
+| `on_conflict`      |          | enum         | `DO_UPDATE` | Whether to make changes if domains already exist. If set to DO_NOTHING, `semantics` setting is irrelevant. |
 
 For transformer behaviour on `replace_existing` and `semantics`, please refer section [Relationship Between replace_existing And semantics](#relationship-between-replace_existing-and-semantics).
 
@@ -191,13 +192,14 @@ transformers:
 
 ## Pattern Add Dataset ownership 
 ### Config Details
-| Field              | Required | Type                 | Default     | Description                                                                             |
-|--------------------|----------|----------------------|-------------|-----------------------------------------------------------------------------------------|
-| `owner_pattern`    | ✅        | map[regx, list[urn]] |             | entity urn with regular expression and list of owners urn apply to matching entity urn. |
-| `ownership_type`   |          | string               | "DATAOWNER" | ownership type of the owners (either as enum or ownership type urn)                     |
-| `replace_existing` |          | boolean              | `false`     | Whether to remove owners from entity sent by ingestion source.                          |
-| `semantics`        |          | enum                 | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS.                        |
-| `is_container`     |          | bool    | `false`    | Whether to also consider a container or not. If true, then ownership will be attached to both the dataset and its container. |
+| Field              | Required | Type                 | Default     | Description                                                                                                                  |
+|--------------------|----------|----------------------|-------------|------------------------------------------------------------------------------------------------------------------------------|
+| `owner_pattern`    | ✅        | map[regx, list[urn]] |             | entity urn with regular expression and list of owners urn apply to matching entity urn.                                      |
+| `ownership_type`   |          | string               | "DATAOWNER" | ownership type of the owners (either as enum or ownership type urn)                                                          |
+| `replace_existing` |          | boolean              | `false`     | Whether to remove owners from entity sent by ingestion source.                                                               |
+| `semantics`        |          | enum                 | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS.                                                             |
+| `is_container`     |          | bool                 | `false`     | Whether to also consider a container or not. If true, then ownership will be attached to both the dataset and its container. |
+| `on_conflict`      |          | enum                 | `DO_UPDATE` | Whether to make changes if domains already exist. If set to DO_NOTHING, `semantics` setting is irrelevant.                   |
 
 let’s suppose we’d like to append a series of users who we know to own a different dataset from a data source but aren't detected during normal ingestion. To do so, we can use the `pattern_add_dataset_ownership` module that’s included in the ingestion framework.  This will match the pattern to `urn` of the dataset and assign the respective owners.
 
diff --git a/metadata-ingestion/docs/transformer/intro.md b/metadata-ingestion/docs/transformer/intro.md
index 0a766f7fd747d..85ba42fca4674 100644
--- a/metadata-ingestion/docs/transformer/intro.md
+++ b/metadata-ingestion/docs/transformer/intro.md
@@ -26,6 +26,8 @@ DataHub provided transformers for dataset are:
 - [Simple Add Dataset ownership](./dataset_transformer.md#simple-add-dataset-ownership)
 - [Pattern Add Dataset ownership](./dataset_transformer.md#pattern-add-dataset-ownership)
 - [Simple Remove Dataset ownership](./dataset_transformer.md#simple-remove-dataset-ownership)
+- [Extract Ownership from Tags](./dataset_transformer.md#extract-ownership-from-tags)
+- [Clean suffix prefix from Ownership](./dataset_transformer.md#clean-suffix-prefix-from-ownership)
 - [Mark Dataset Status](./dataset_transformer.md#mark-dataset-status)
 - [Simple Add Dataset globalTags](./dataset_transformer.md#simple-add-dataset-globaltags)
 - [Pattern Add Dataset globalTags](./dataset_transformer.md#pattern-add-dataset-globaltags)
@@ -33,9 +35,14 @@ DataHub provided transformers for dataset are:
 - [Set Dataset browsePath](./dataset_transformer.md#set-dataset-browsepath)
 - [Simple Add Dataset glossaryTerms](./dataset_transformer.md#simple-add-dataset-glossaryterms)
 - [Pattern Add Dataset glossaryTerms](./dataset_transformer.md#pattern-add-dataset-glossaryterms)
+- [Add Dataset globalTags](./dataset_transformer.md#add-dataset-globaltags)
 - [Pattern Add Dataset Schema Field glossaryTerms](./dataset_transformer.md#pattern-add-dataset-schema-field-glossaryterms)
 - [Pattern Add Dataset Schema Field globalTags](./dataset_transformer.md#pattern-add-dataset-schema-field-globaltags)
 - [Simple Add Dataset datasetProperties](./dataset_transformer.md#simple-add-dataset-datasetproperties)
 - [Add Dataset datasetProperties](./dataset_transformer.md#add-dataset-datasetproperties)
 - [Simple Add Dataset domains](./dataset_transformer.md#simple-add-dataset-domains)
 - [Pattern Add Dataset domains](./dataset_transformer.md#pattern-add-dataset-domains)
+- [Domain Mapping Based on Tags](./dataset_transformer.md#domain-mapping-based-on-tags)  
+- [Simple Add Dataset dataProduct ](./dataset_transformer.md#simple-add-dataset-dataproduct)
+- [Pattern Add Dataset dataProduct](./dataset_transformer.md#pattern-add-dataset-dataproduct)
+- [Add Dataset dataProduct](./dataset_transformer.md#add-dataset-dataproduct)
diff --git a/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml b/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml
index d0fea81748da5..a1adec58b7b5e 100644
--- a/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml
+++ b/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml
@@ -1,4 +1,4 @@
-version: 1
+version: "1"
 source: DataHub
 owners:
   users:
@@ -68,4 +68,4 @@ nodes:
           - name: Auto Loan
             description: "A type of loan used to finance the purchase of a vehicle, with the vehicle serving as collateral for the loan."
           - name: Interest Rate
-            description: "The rate at which interest is charged on a loan or paid on an investment, expressed as a percentage of the principal amount."
\ No newline at end of file
+            description: "The rate at which interest is charged on a loan or paid on an investment, expressed as a percentage of the principal amount."
diff --git a/metadata-ingestion/examples/bootstrap_data/business_glossary.yml b/metadata-ingestion/examples/bootstrap_data/business_glossary.yml
index 327246863b0ab..20d1011b96689 100644
--- a/metadata-ingestion/examples/bootstrap_data/business_glossary.yml
+++ b/metadata-ingestion/examples/bootstrap_data/business_glossary.yml
@@ -1,4 +1,4 @@
-version: 1
+version: "1"
 source: DataHub
 owners:
   users:
@@ -11,20 +11,20 @@ nodes:
       - label: Wiki link for classification
         url: "https://en.wikipedia.org/wiki/Classification"
     custom_properties:
-      is_confidential: true
+      is_confidential: "true"
     terms:
       - name: Sensitive
         description: Sensitive Data
         custom_properties:
-          is_confidential: false
+          is_confidential: "false"
       - name: Confidential
         description: Confidential Data
         custom_properties:
-          is_confidential: true
+          is_confidential: "true"
       - name: HighlyConfidential
         description: Highly Confidential Data
         custom_properties:
-          is_confidential: true
+          is_confidential: "true"
         domain: Marketing
   - name: PersonalInformation
     description: All terms related to personal information
@@ -72,7 +72,7 @@ nodes:
           - Shipping.CountryCode
           - Shipping.StreetAddress
         custom_properties:
-          is_used_for_compliance_tracking: true
+          is_used_for_compliance_tracking: "true"
         knowledge_links:
           - url: "https://en.wikipedia.org/wiki/Address"
             label: Wiki link
diff --git a/metadata-ingestion/examples/cli_usage/gen_schemas.py b/metadata-ingestion/examples/cli_usage/gen_schemas.py
index 2fd4683347a3b..80b2c6712977a 100644
--- a/metadata-ingestion/examples/cli_usage/gen_schemas.py
+++ b/metadata-ingestion/examples/cli_usage/gen_schemas.py
@@ -28,7 +28,6 @@ class CorpGroupFile(BaseModel):
 
 
 with open("user/user.dhub.yaml_schema.json", "w") as fp:
-
     fp.write(json.dumps(CorpUserFile.schema(), indent=4))
 
 with open("group/group.dhub.yaml_schema.json", "w") as fp:
diff --git a/metadata-ingestion/examples/library/dataset_add_custom_properties_patch.py b/metadata-ingestion/examples/library/dataset_add_custom_properties_patch.py
new file mode 100644
index 0000000000000..7231461fea322
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_add_custom_properties_patch.py
@@ -0,0 +1,19 @@
+from datahub.emitter.mce_builder import make_dataset_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD")
+
+# Create Dataset Patch to Add Custom Properties
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.add_custom_property("cluster_name", "datahubproject.acryl.io")
+patch_builder.add_custom_property("retention_time", "2 years")
+patch_mcps = patch_builder.build()
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/library/dataset_add_glossary_term_patch.py b/metadata-ingestion/examples/library/dataset_add_glossary_term_patch.py
new file mode 100644
index 0000000000000..d0b9a866fde61
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_add_glossary_term_patch.py
@@ -0,0 +1,22 @@
+from datahub.emitter.mce_builder import make_dataset_urn, make_term_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.metadata.schema_classes import GlossaryTermAssociationClass
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(
+    platform="snowflake", name="fct_users_created", env="PROD"
+)
+
+# Create Dataset Patch to Add + Remove Term for 'profile_id' column
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.add_term(GlossaryTermAssociationClass(make_term_urn("term-to-add-id")))
+patch_builder.remove_term(make_term_urn("term-to-remove-id"))
+patch_mcps = patch_builder.build()
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/library/dataset_add_owner_patch.py b/metadata-ingestion/examples/library/dataset_add_owner_patch.py
new file mode 100644
index 0000000000000..8d3130c09c4bb
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_add_owner_patch.py
@@ -0,0 +1,24 @@
+from datahub.emitter.mce_builder import make_dataset_urn, make_group_urn, make_user_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.metadata.schema_classes import OwnerClass, OwnershipTypeClass
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(
+    platform="snowflake", name="fct_users_created", env="PROD"
+)
+
+# Create Dataset Patch to Add + Remove Owners
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.add_owner(
+    OwnerClass(make_user_urn("user-to-add-id"), OwnershipTypeClass.TECHNICAL_OWNER)
+)
+patch_builder.remove_owner(make_group_urn("group-to-remove-id"))
+patch_mcps = patch_builder.build()
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/library/dataset_add_properties.py b/metadata-ingestion/examples/library/dataset_add_properties.py
deleted file mode 100644
index b72aac5b82800..0000000000000
--- a/metadata-ingestion/examples/library/dataset_add_properties.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import logging
-from typing import Union
-
-from datahub.configuration.kafka import KafkaProducerConnectionConfig
-from datahub.emitter.kafka_emitter import DatahubKafkaEmitter, KafkaEmitterConfig
-from datahub.emitter.mce_builder import make_dataset_urn
-from datahub.emitter.rest_emitter import DataHubRestEmitter
-from datahub.specific.dataset import DatasetPatchBuilder
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-# Get an emitter, either REST or Kafka, this example shows you both
-def get_emitter() -> Union[DataHubRestEmitter, DatahubKafkaEmitter]:
-    USE_REST_EMITTER = True
-    if USE_REST_EMITTER:
-        gms_endpoint = "http://localhost:8080"
-        return DataHubRestEmitter(gms_server=gms_endpoint)
-    else:
-        kafka_server = "localhost:9092"
-        schema_registry_url = "http://localhost:8081"
-        return DatahubKafkaEmitter(
-            config=KafkaEmitterConfig(
-                connection=KafkaProducerConnectionConfig(
-                    bootstrap=kafka_server, schema_registry_url=schema_registry_url
-                )
-            )
-        )
-
-
-dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD")
-
-with get_emitter() as emitter:
-    for patch_mcp in (
-        DatasetPatchBuilder(dataset_urn)
-        .add_custom_property("cluster_name", "datahubproject.acryl.io")
-        .add_custom_property("retention_time", "2 years")
-        .build()
-    ):
-        emitter.emit(patch_mcp)
-
-
-log.info(f"Added cluster_name, retention_time properties to dataset {dataset_urn}")
diff --git a/metadata-ingestion/examples/library/dataset_add_remove_custom_properties_patch.py b/metadata-ingestion/examples/library/dataset_add_remove_custom_properties_patch.py
new file mode 100644
index 0000000000000..c1db9c91d13ec
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_add_remove_custom_properties_patch.py
@@ -0,0 +1,19 @@
+from datahub.emitter.mce_builder import make_dataset_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD")
+
+# Create Dataset Patch to Add + Remove Custom Properties
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.add_custom_property("cluster_name", "datahubproject.acryl.io")
+patch_builder.remove_custom_property("retention_time")
+patch_mcps = patch_builder.build()
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/library/dataset_add_remove_properties.py b/metadata-ingestion/examples/library/dataset_add_remove_properties.py
deleted file mode 100644
index 7109c0264f971..0000000000000
--- a/metadata-ingestion/examples/library/dataset_add_remove_properties.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import logging
-from typing import Union
-
-from datahub.configuration.kafka import KafkaProducerConnectionConfig
-from datahub.emitter.kafka_emitter import DatahubKafkaEmitter, KafkaEmitterConfig
-from datahub.emitter.mce_builder import make_dataset_urn
-from datahub.emitter.rest_emitter import DataHubRestEmitter
-from datahub.specific.dataset import DatasetPatchBuilder
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-# Get an emitter, either REST or Kafka, this example shows you both
-def get_emitter() -> Union[DataHubRestEmitter, DatahubKafkaEmitter]:
-    USE_REST_EMITTER = True
-    if USE_REST_EMITTER:
-        gms_endpoint = "http://localhost:8080"
-        return DataHubRestEmitter(gms_server=gms_endpoint)
-    else:
-        kafka_server = "localhost:9092"
-        schema_registry_url = "http://localhost:8081"
-        return DatahubKafkaEmitter(
-            config=KafkaEmitterConfig(
-                connection=KafkaProducerConnectionConfig(
-                    bootstrap=kafka_server, schema_registry_url=schema_registry_url
-                )
-            )
-        )
-
-
-dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD")
-
-with get_emitter() as emitter:
-    for patch_mcp in (
-        DatasetPatchBuilder(dataset_urn)
-        .add_custom_property("cluster_name", "datahubproject.acryl.io")
-        .remove_custom_property("retention_time")
-        .build()
-    ):
-        emitter.emit(patch_mcp)
-
-
-log.info(
-    f"Added cluster_name property, removed retention_time property from dataset {dataset_urn}"
-)
diff --git a/metadata-ingestion/examples/library/dataset_add_structured_properties.py b/metadata-ingestion/examples/library/dataset_add_structured_properties.py
deleted file mode 100644
index fc2c379340592..0000000000000
--- a/metadata-ingestion/examples/library/dataset_add_structured_properties.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import logging
-
-from datahub.emitter.mce_builder import make_dataset_urn
-from datahub.emitter.rest_emitter import DataHubRestEmitter
-from datahub.specific.dataset import DatasetPatchBuilder
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-# Create rest emitter
-rest_emitter = DataHubRestEmitter(gms_server="http://localhost:8080")
-
-dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD")
-
-
-for patch_mcp in (
-    DatasetPatchBuilder(dataset_urn)
-    .add_structured_property("io.acryl.dataManagement.replicationSLA", 12)
-    .build()
-):
-    rest_emitter.emit(patch_mcp)
-
-
-log.info(f"Added cluster_name, retention_time properties to dataset {dataset_urn}")
diff --git a/metadata-ingestion/examples/library/dataset_add_structured_properties_patch.py b/metadata-ingestion/examples/library/dataset_add_structured_properties_patch.py
new file mode 100644
index 0000000000000..ef72ed58a4b82
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_add_structured_properties_patch.py
@@ -0,0 +1,23 @@
+from datahub.emitter.mce_builder import make_dataset_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD")
+
+# Create Dataset Patch to Add and Remove Structured Properties
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.add_structured_property(
+    "urn:li:structuredProperty:retentionTimeInDays", 12
+)
+patch_builder.remove_structured_property(
+    "urn:li:structuredProperty:customClassification"
+)
+patch_mcps = patch_builder.build()
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/library/dataset_add_tag_patch.py b/metadata-ingestion/examples/library/dataset_add_tag_patch.py
new file mode 100644
index 0000000000000..0bc644d6865f6
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_add_tag_patch.py
@@ -0,0 +1,22 @@
+from datahub.emitter.mce_builder import make_dataset_urn, make_tag_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.metadata.schema_classes import TagAssociationClass
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(
+    platform="snowflake", name="fct_users_created", env="PROD"
+)
+
+# Create Dataset Patch
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.add_tag(TagAssociationClass(make_tag_urn("tag-to-add-id")))
+patch_builder.remove_tag("urn:li:tag:tag-to-remove-id")
+patch_mcps = patch_builder.build()
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/library/dataset_add_upstream_lineage_patch.py b/metadata-ingestion/examples/library/dataset_add_upstream_lineage_patch.py
new file mode 100644
index 0000000000000..0b4e5e39bf627
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_add_upstream_lineage_patch.py
@@ -0,0 +1,62 @@
+from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.metadata.schema_classes import (
+    DatasetLineageTypeClass,
+    FineGrainedLineageClass,
+    FineGrainedLineageUpstreamTypeClass,
+    UpstreamClass,
+)
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(
+    platform="snowflake", name="fct_users_created", env="PROD"
+)
+upstream_to_remove_urn = make_dataset_urn(
+    platform="s3", name="fct_users_old", env="PROD"
+)
+upstream_to_add_urn = make_dataset_urn(platform="s3", name="fct_users_new", env="PROD")
+
+# Create Dataset Patch to Add & Remove Upstream Lineage Edges
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.remove_upstream_lineage(upstream_to_remove_urn)
+patch_builder.add_upstream_lineage(
+    UpstreamClass(upstream_to_add_urn, DatasetLineageTypeClass.TRANSFORMED)
+)
+
+# ...And also include schema field lineage
+upstream_field_to_add_urn = make_schema_field_urn(upstream_to_add_urn, "profile_id")
+downstream_field_to_add_urn = make_schema_field_urn(dataset_urn, "profile_id")
+
+patch_builder.add_fine_grained_upstream_lineage(
+    FineGrainedLineageClass(
+        FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+        FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+        [upstream_field_to_add_urn],
+        [downstream_field_to_add_urn],
+    )
+)
+
+upstream_field_to_remove_urn = make_schema_field_urn(
+    upstream_to_remove_urn, "profile_id"
+)
+downstream_field_to_remove_urn = make_schema_field_urn(dataset_urn, "profile_id")
+
+patch_builder.remove_fine_grained_upstream_lineage(
+    FineGrainedLineageClass(
+        FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+        FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+        [upstream_field_to_remove_urn],
+        [downstream_field_to_remove_urn],
+    )
+)
+
+patch_mcps = patch_builder.build()
+
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/library/dataset_field_add_glossary_term_patch.py b/metadata-ingestion/examples/library/dataset_field_add_glossary_term_patch.py
new file mode 100644
index 0000000000000..3f8da2c143c92
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_field_add_glossary_term_patch.py
@@ -0,0 +1,26 @@
+from datahub.emitter.mce_builder import make_dataset_urn, make_term_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.metadata.schema_classes import GlossaryTermAssociationClass
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(
+    platform="snowflake", name="fct_users_created", env="PROD"
+)
+
+# Create Dataset Patch to Add + Remove Term for 'profile_id' column
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.for_field("profile_id").add_term(
+    GlossaryTermAssociationClass(make_term_urn("term-to-add-id"))
+)
+patch_builder.for_field("profile_id").remove_term(
+    "urn:li:glossaryTerm:term-to-remove-id"
+)
+patch_mcps = patch_builder.build()
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/library/dataset_field_add_tag_patch.py b/metadata-ingestion/examples/library/dataset_field_add_tag_patch.py
new file mode 100644
index 0000000000000..3075cac5320ae
--- /dev/null
+++ b/metadata-ingestion/examples/library/dataset_field_add_tag_patch.py
@@ -0,0 +1,24 @@
+from datahub.emitter.mce_builder import make_dataset_urn, make_tag_urn
+from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
+from datahub.metadata.schema_classes import TagAssociationClass
+from datahub.specific.dataset import DatasetPatchBuilder
+
+# Create DataHub Client
+datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080"))
+
+# Create Dataset URN
+dataset_urn = make_dataset_urn(
+    platform="snowflake", name="fct_users_created", env="PROD"
+)
+
+# Create Dataset Patch to Add + Remove Tag for 'profile_id' column
+patch_builder = DatasetPatchBuilder(dataset_urn)
+patch_builder.for_field("profile_id").add_tag(
+    TagAssociationClass(make_tag_urn("tag-to-add-id"))
+)
+patch_builder.for_field("profile_id").remove_tag("urn:li:tag:tag-to-remove-id")
+patch_mcps = patch_builder.build()
+
+# Emit Dataset Patch
+for patch_mcp in patch_mcps:
+    datahub_client.emit(patch_mcp)
diff --git a/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml
index 86f4898d9d502..4210d0599a215 100644
--- a/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml
+++ b/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml
@@ -41,6 +41,23 @@ source:
     #  deny:
     #  - "*.*.*"
     #storage_project_id: project-id-1234567
+    ## Lineage with GCS Source
+    # include_column_lineage_with_gcs: true/false
+    # gcs_lineage_config:
+    #   path_specs:
+    #     - include: "gs://my-bucket/foo/tests/bar.avro"
+    #     - include: "gs://my-bucket/foo/tests/*.*"
+    #     - include: "gs://my-bucket/foo/tests/{table}/*.avro"
+    #     - include: "gs://my-bucket/foo/tests/{table}/*/*.avro"
+    #     - include: "gs://my-bucket/foo/tests/{table}/*.*"
+    #     - include: "gs://my-bucket/{dept}/tests/{table}/*.avro"
+    #     - include: "gs://my-bucket/{dept}/tests/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.avro"
+    #     - include: "gs://my-bucket/{dept}/tests/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.avro"
+    #     - include: "gs://my-bucket/{dept}/tests/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.*"
+    #     - include: "gs://my-bucket/*/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.*"
+    #     - include: "gs://my-bucket/*/*/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.*"
+    # strip_urls: false
+
 
 ## see https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for complete documentation
 sink:
diff --git a/metadata-ingestion/examples/structured_properties/structured_properties.yaml b/metadata-ingestion/examples/structured_properties/structured_properties.yaml
index 5c7ce47ba3b8a..0230fbcc61f11 100644
--- a/metadata-ingestion/examples/structured_properties/structured_properties.yaml
+++ b/metadata-ingestion/examples/structured_properties/structured_properties.yaml
@@ -7,7 +7,7 @@
   entity_types:
     - dataset # or urn:li:entityType:datahub.dataset
     - dataFlow
-  description: "Retention Time is used to figure out how long to retain records in a dataset"
+  description: 'Retention Time is used to figure out how long to retain records in a dataset'
   allowed_values:
     - value: 30
       description: 30 days, usually reserved for datasets that are ephemeral and contain pii
@@ -18,7 +18,7 @@
 - id: io.acryl.dataManagement.replicationSLA
   type: number
   display_name: Replication SLA
-  description: "SLA for how long data can be delayed before replicating to the destination cluster"
+  description: 'SLA for how long data can be delayed before replicating to the destination cluster'
   entity_types:
     - dataset
 - id: io.acryl.dataManagement.deprecationDate
diff --git a/metadata-ingestion/pyproject.toml b/metadata-ingestion/pyproject.toml
index 2b6c87926c6c4..94e06fd53a70e 100644
--- a/metadata-ingestion/pyproject.toml
+++ b/metadata-ingestion/pyproject.toml
@@ -14,7 +14,7 @@ target-version = ['py37', 'py38', 'py39', 'py310']
 [tool.isort]
 combine_as_imports = true
 indent = '    '
-known_future_library = ['__future__', 'datahub.utilities._markupsafe_compat', 'datahub_provider._airflow_compat']
+known_future_library = ['__future__', 'datahub.utilities._markupsafe_compat', 'datahub.sql_parsing._sqlglot_patch']
 profile = 'black'
 sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER'
 skip_glob = 'src/datahub/metadata'
diff --git a/metadata-ingestion/scripts/avro_codegen.py b/metadata-ingestion/scripts/avro_codegen.py
index 5be8b6ed4cc21..e5792da32fb5d 100644
--- a/metadata-ingestion/scripts/avro_codegen.py
+++ b/metadata-ingestion/scripts/avro_codegen.py
@@ -361,9 +361,6 @@ def write_urn_classes(key_aspects: List[dict], urn_dir: Path) -> None:
 
     for aspect in key_aspects:
         entity_type = aspect["Aspect"]["keyForEntity"]
-        if aspect["Aspect"]["entityCategory"] == "internal":
-            continue
-
         code += generate_urn_class(entity_type, aspect)
 
     (urn_dir / "urn_defs.py").write_text(code)
@@ -772,7 +769,7 @@ def generate(
 import importlib
 from typing import TYPE_CHECKING
 
-from datahub._codegen.aspect import _Aspect
+from datahub._codegen.aspect import _Aspect as _Aspect
 from datahub.utilities.docs_build import IS_SPHINX_BUILD
 from datahub.utilities._custom_package_loader import get_custom_models_package
 
@@ -805,7 +802,7 @@ def generate(
 
 from datahub.utilities.docs_build import IS_SPHINX_BUILD
 from datahub.utilities._custom_package_loader import get_custom_urns_package
-from datahub.utilities.urns._urn_base import Urn  # noqa: F401
+from datahub.utilities.urns._urn_base import Urn as Urn  # noqa: F401
 
 _custom_package_path = get_custom_urns_package()
 
diff --git a/metadata-ingestion/scripts/datahub_preflight.sh b/metadata-ingestion/scripts/datahub_preflight.sh
deleted file mode 100755
index 9676964f4d49d..0000000000000
--- a/metadata-ingestion/scripts/datahub_preflight.sh
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/bin/bash -e
-
-#From https://stackoverflow.com/questions/4023830/how-to-compare-two-strings-in-dot-separated-version-format-in-bash
-verlte() {
-  [  "$1" == "$(echo -e "$1\n$2" | sort -V | head -n1)" ]
-}
-
-brew_install() {
-    package=${1}
-    required_version=${2}
-    printf '\n🔎 Checking if %s installed\n' "${package}"
-    version=$(brew list --version|grep "$1"|awk '{ print $2 }')
-
-    if [ -n "${version}" ]; then
-      if [ -n "$2" ] && ! verlte "${required_version}" "${version}"; then
-        printf '🔽 %s is installed but its version %s is lower than the required %s\n' "${package}" "${version}" "${required_version}. Updating version..."
-        brew update && brew upgrade "$1" && printf '✅ %s is installed\n' "${package}"
-      else
-        printf '✅ %s is already installed\n' "${package} with version ${version}"
-      fi
-    else
-        brew install "$1" && printf '✅ %s is installed\n' "${package}"
-    fi
-}
-
-arm64_darwin_preflight() {
-  printf "✨ Creating/activating Virtual Environment\n"
-  python3 -m venv venv
-  source venv/bin/activate
-
-  printf "🔎 Checking if Scipy installed\n"
-  if pip list | grep -F scipy; then
-  	printf "✅ Scipy already installed\n"
-  else
-  	printf "Scipy not installed\n"
-  	printf "⛅ Installing prerequisities for scipy"
-  	brew install openblas
-  	OPENBLAS="$(brew --prefix openblas)"
-  	export OPENBLAS
-  	##preinstall numpy and pythran from source
-  	pip3 uninstall -y numpy pythran
-  	pip3 install cython pybind11
-  	pip3 install --no-use-pep517 numpy
-  	pip3 install pythran
-  	pip3 install --no-use-pep517 scipy
-  fi
-
-  brew_install "openssl@1.1"
-  brew install "postgresql@14"
-
-  # postgresql installs libs in a strange way
-  # we first symlink /opt/postgresql@14 to /opt/postgresql
-  if [ ! -z $(brew --prefix)/opt/postgresql ]; then
-    printf "✨ Symlinking postgresql@14 to postgresql\n"
-    ln -sf $(brew --prefix postgresql@14) $(brew --prefix)/opt/postgresql
-  fi
-  # we then symlink all libs under /opt/postgresql@14/lib/postgresql@14 to /opt/postgresql@14/lib
-  if [ ! -z $(brew --prefix postgresql@14)/lib/postgresql@14 ]; then
-    printf "✨ Patching up libs in $(brew --prefix postgresql@14)/lib/postgresql@14)\n"
-    ln -sf $(brew --prefix postgresql@14)/lib/postgresql@14/* $(brew --prefix postgresql@14)/lib/
-  fi
-
-  printf "\e[38;2;0;255;0m✅ Done\e[38;2;255;255;255m\n"
-
-  printf "✨ Setting up environment variable:\n"
-  GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1
-  export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL
-  GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1
-  export GRPC_PYTHON_BUILD_SYSTEM_ZLIB
-  CPPFLAGS="-I$(brew --prefix openssl@1.1)/include"
-  export CPPFLAGS
-  LDFLAGS="-L$(brew --prefix openssl@1.1)/lib"
-  export LDFLAGS
-
-cat << EOF
-  export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1
-  export GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1
-  export CPPFLAGS="-I$(brew --prefix openssl@1.1)/include"
-  export LDFLAGS="-L$(brew --prefix openssl@1.1)/lib -L$(brew --prefix postgresql@14)/lib/postgresql@14"
-
-EOF
-
-  if pip list | grep -F confluent-kafka; then
-    printf "✅ confluent-kafka already installed\n"
-  else
-    pip3 install confluent-kafka
-  fi
-
-  printf "✨ Setting up prerequisities\n"
-  # none for now, since jq was removed
-
-  printf "\e[38;2;0;255;0m✅ Done\e[38;2;255;255;255m\n"
-}
-
-
-printf "🔎 Checking if current directory is metadata-ingestion folder\n"
-if [ "$(basename "$(pwd)")"	 != "metadata-ingestion" ]; then
-	printf "💥 You should run this script in Datahub\'s metadata-ingestion folder but your folder is %s\n" "$(pwd)"
-	exit 123
-fi
-printf '✅ Current folder is metadata-ingestion (%s) folder\n' "$(pwd)"
-if [[ $(uname -m) == 'arm64' && $(uname) == 'Darwin' ]]; then
-  printf "👟 Running preflight for m1 mac\n"
-  arm64_darwin_preflight
-fi
-
-
-printf "\n\e[38;2;0;255;0m✅ Preflight was successful\e[38;2;255;255;255m\n"
diff --git a/metadata-ingestion/scripts/docgen.py b/metadata-ingestion/scripts/docgen.py
index 797a2f698c2f4..402cd8a814199 100644
--- a/metadata-ingestion/scripts/docgen.py
+++ b/metadata-ingestion/scripts/docgen.py
@@ -918,7 +918,7 @@ def generate(
 <img width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/lineage/lineage-tab.png" />
 </p>
 
-By default, The UI shows the latest version of the lineage. The time picker can be used to filter out edges within the latest version to exclude those that were last updated outside of the time window. Selecting time windows in the patch will not show you historical lineages. It will only filter the view of the latest version of the lineage.
+By default, the UI shows the latest version of the lineage. The time picker can be used to filter out edges within the latest version to exclude those that were last updated outside of the time window. Selecting time windows in the patch will not show you historical lineages. It will only filter the view of the latest version of the lineage.
 
 <p align="center">
 <img width="80%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/lineage/lineage-view.png" />
@@ -969,7 +969,7 @@ def generate(
 ## Lineage Support
 
 DataHub supports **[automatic table- and column-level lineage detection](#automatic-lineage-extraction-support)** from BigQuery, Snowflake, dbt, Looker, PowerBI, and 20+ modern data tools. 
-For data tools with limited native lineage tracking, **DataHub's SQL Parser** detects lineage with 97–99% accuracy, ensuring teams will have high quality lineage graphs across all corners of their data stack.
+For data tools with limited native lineage tracking, [**DataHub's SQL Parser**](../../lineage/sql_parsing.md) detects lineage with 97-99% accuracy, ensuring teams will have high quality lineage graphs across all corners of their data stack.
 
 ### Types of Lineage Connections
 
diff --git a/metadata-ingestion/scripts/modeldocgen.py b/metadata-ingestion/scripts/modeldocgen.py
index ee5f06cb801ba..998947e5caa95 100644
--- a/metadata-ingestion/scripts/modeldocgen.py
+++ b/metadata-ingestion/scripts/modeldocgen.py
@@ -14,7 +14,11 @@
 import click
 
 from datahub.configuration.common import ConfigEnum, PermissiveConfigModel
-from datahub.emitter.mce_builder import make_data_platform_urn, make_dataset_urn
+from datahub.emitter.mce_builder import (
+    make_data_platform_urn,
+    make_dataset_urn,
+    make_schema_field_urn,
+)
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.rest_emitter import DatahubRestEmitter
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
@@ -442,10 +446,10 @@ def strip_types(field_path: str) -> str:
                                 name=relnship_name,
                                 foreignDataset=foreign_dataset_urn,
                                 foreignFields=[
-                                    f"urn:li:schemaField:({foreign_dataset_urn}, urn)"
+                                    make_schema_field_urn(foreign_dataset_urn, "urn")
                                 ],
                                 sourceFields=[
-                                    f"urn:li:schemaField:({source_dataset_urn},{f_field.fieldPath})"
+                                    make_schema_field_urn(source_dataset_urn, f_field.fieldPath)
                                 ],
                             )
                             foreign_keys.append(fkey)
diff --git a/metadata-ingestion/setup.cfg b/metadata-ingestion/setup.cfg
index c095420e4e3f3..057779bc87c62 100644
--- a/metadata-ingestion/setup.cfg
+++ b/metadata-ingestion/setup.cfg
@@ -31,7 +31,7 @@ exclude =
     __pycache__
 per-file-ignores =
     # imported but unused
-    __init__.py: F401
+    __init__.py: F401, I250
 ban-relative-imports = true
 
 [mypy]
@@ -53,6 +53,14 @@ disallow_untyped_defs = no
 # try to be a bit more strict in certain areas of the codebase
 [mypy-datahub.*]
 ignore_missing_imports = no
+implicit_reexport = no
+[mypy-datahub.metadata.*]
+# TODO: Remove this once all the code has been updated.
+implicit_reexport = yes
+[mypy-datahub.ingestion.*]
+# TODO: Remove this once all the code has been updated.
+implicit_reexport = yes
+
 [mypy-datahub_provider.*]
 ignore_missing_imports = no
 [mypy-tests.*]
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index c4b5496655349..af6266e967af9 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -14,8 +14,8 @@
 )
 
 base_requirements = {
-    # Typing extension should be >=3.10.0.2 ideally but we can't restrict due to a Airflow 2.1 dependency conflict.
-    "typing_extensions>=3.7.4.3",
+    # Our min version of typing_extensions is somewhat constrained by Airflow.
+    "typing_extensions>=3.10.0.2",
     # Actual dependencies.
     "typing-inspect",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
@@ -42,7 +42,6 @@
     "python-dateutil>=2.8.0",
     "tabulate",
     "progressbar2",
-    "termcolor>=1.0.0",
     "psutil>=5.8.0",
     "Deprecated",
     "humanfriendly",
@@ -77,7 +76,7 @@
     # now provide prebuilt wheels for most platforms, including M1 Macs and
     # Linux aarch64 (e.g. Docker's linux/arm64). Installing confluent_kafka
     # from source remains a pain.
-    "confluent_kafka>=1.9.0",
+    "confluent_kafka[schemaregistry]>=1.9.0",
     # We currently require both Avro libraries. The codegen uses avro-python3 (above)
     # schema parsers at runtime for generating and reading JSON into Python objects.
     # At the same time, we use Kafka's AvroSerializer, which internally relies on
@@ -99,9 +98,11 @@
 }
 
 sqlglot_lib = {
-    # Using an Acryl fork of sqlglot.
+    # We heavily monkeypatch sqlglot.
+    # Prior to the patching, we originally maintained an acryl-sqlglot fork:
     # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main?expand=1
-    "acryl-sqlglot[rs]==25.20.2.dev6",
+    "sqlglot[rs]==25.32.1",
+    "patchy==2.8.0",
 }
 
 classification_lib = {
@@ -122,6 +123,10 @@
     "more_itertools",
 }
 
+cachetools_lib = {
+    "cachetools",
+}
+
 sql_common = (
     {
         # Required for all SQL sources.
@@ -136,22 +141,24 @@
         # https://github.com/great-expectations/great_expectations/pull/5382/files
         # datahub does not depend on traitlets directly but great expectations does.
         # https://github.com/ipython/traitlets/issues/741
-        "traitlets<5.2.2",
+        "traitlets!=5.2.2",
+        # GE depends on IPython - we have no direct dependency on it.
+        # IPython 8.22.0 added a dependency on traitlets 5.13.x, but only declared a
+        # version requirement of traitlets>5.
+        # See https://github.com/ipython/ipython/issues/14352.
+        # This issue was fixed by https://github.com/ipython/ipython/pull/14353,
+        # which first appeared in IPython 8.22.1.
+        # As such, we just need to avoid that version in order to get the
+        # dependencies that we need. IPython probably should've yanked 8.22.0.
+        "IPython!=8.22.0",
         "greenlet",
+        *cachetools_lib,
     }
     | usage_common
     | sqlglot_lib
     | classification_lib
 )
 
-sqllineage_lib = {
-    "sqllineage==1.3.8",
-    # We don't have a direct dependency on sqlparse but it is a dependency of sqllineage.
-    # There have previously been issues from not pinning sqlparse, so it's best to pin it.
-    # Related: https://github.com/reata/sqllineage/issues/361 and https://github.com/reata/sqllineage/pull/360
-    "sqlparse==0.4.4",
-}
-
 aws_common = {
     # AWS Python SDK
     "boto3",
@@ -167,7 +174,7 @@
 
 looker_common = {
     # Looker Python SDK
-    "looker-sdk==23.0.0",
+    "looker-sdk>=23.0.0",
     # This version of lkml contains a fix for parsing lists in
     # LookML files with spaces between an item and the following comma.
     # See https://github.com/joshtemple/lkml/issues/73.
@@ -186,6 +193,7 @@
     "google-cloud-resource-manager",
     "more-itertools>=8.12.0",
     "sqlalchemy-bigquery>=1.4.1",
+    *path_spec_common,
 }
 
 clickhouse_common = {
@@ -200,7 +208,6 @@
     "sqlalchemy-redshift>=0.8.3",
     "GeoAlchemy2",
     "redshift-connector>=2.1.0",
-    *sqllineage_lib,
     *path_spec_common,
 }
 
@@ -213,7 +220,7 @@
     "pandas",
     "cryptography",
     "msal",
-    "cachetools",
+    *cachetools_lib,
 } | classification_lib
 
 trino = {
@@ -238,11 +245,12 @@
     # Instead, we put the fix in our PyHive fork, so no thrift pin is needed.
 }
 
-microsoft_common = {"msal>=1.22.0"}
+microsoft_common = {"msal>=1.24.0"}
 
 iceberg_common = {
     # Iceberg Python SDK
-    "pyiceberg>=0.4,<0.7",
+    # Kept at 0.4.0 due to higher versions requiring pydantic>2, as soon as we are fine with it, bump this dependency
+    "pyiceberg>=0.4.0",
 }
 
 mssql_common = {
@@ -269,6 +277,13 @@
     *path_spec_common,
 }
 
+threading_timeout_common = {
+    "stopit==1.1.2",
+    # stopit uses pkg_resources internally, which means there's an implied
+    # dependency on setuptools.
+    "setuptools",
+}
+
 abs_base = {
     "azure-core==1.29.4",
     "azure-identity>=1.17.1",
@@ -321,6 +336,13 @@
     "Authlib",
 }
 
+superset_common = {
+    "requests",
+    "sqlalchemy",
+    "great_expectations",
+    "greenlet",
+}
+
 # Note: for all of these, framework_common will be added.
 plugins: Dict[str, Set[str]] = {
     # Sink plugins.
@@ -361,11 +383,11 @@
     "azure-ad": set(),
     "bigquery": sql_common
     | bigquery_common
+    | sqlglot_lib
+    | classification_lib
     | {
-        *sqlglot_lib,
         "google-cloud-datacatalog-lineage==0.2.2",
-    }
-    | classification_lib,
+    },
     "bigquery-queries": sql_common | bigquery_common | sqlglot_lib,
     "clickhouse": sql_common | clickhouse_common,
     "clickhouse-usage": sql_common | usage_common | clickhouse_common,
@@ -375,6 +397,7 @@
     "delta-lake": {*data_lake_profiling, *delta_lake},
     "dbt": {"requests"} | dbt_common | aws_common,
     "dbt-cloud": {"requests"} | dbt_common,
+    "dremio": {"requests"} | sql_common,
     "druid": sql_common | {"pydruid>=0.6.2"},
     "dynamodb": aws_common | classification_lib,
     # Starting with 7.14.0 python client is checking if it is connected to elasticsearch client. If its not it throws
@@ -382,6 +405,13 @@
     # https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/release-notes.html#rn-7-14-0
     # https://github.com/elastic/elasticsearch-py/issues/1639#issuecomment-883587433
     "elasticsearch": {"elasticsearch==7.13.4"},
+    "cassandra": {
+        "cassandra-driver>=3.28.0",
+        # We were seeing an error like this `numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject`
+        # with numpy 2.0. This likely indicates a mismatch between scikit-learn and numpy versions.
+        # https://stackoverflow.com/questions/40845304/runtimewarning-numpy-dtype-size-changed-may-indicate-binary-incompatibility
+        "numpy<2",
+    },
     "feast": {
         "feast>=0.34.0,<1",
         "flask-openid>=1.3.0",
@@ -425,9 +455,7 @@
         # It's technically wrong for packages to depend on setuptools. However, it seems mlflow does it anyways.
         "setuptools",
     },
-    "mode": {"requests", "python-liquid", "tenacity>=8.0.1"}
-    | sqllineage_lib
-    | sqlglot_lib,
+    "mode": {"requests", "python-liquid", "tenacity>=8.0.1"} | sqlglot_lib,
     "mongodb": {"pymongo[srv]>=3.11", "packaging"},
     "mssql": sql_common | mssql_common,
     "mssql-odbc": sql_common | mssql_common | {"pyodbc"},
@@ -435,7 +463,7 @@
     # mariadb should have same dependency as mysql
     "mariadb": sql_common | {"pymysql>=1.0.2"},
     "okta": {"okta~=1.7.0", "nest-asyncio"},
-    "oracle": sql_common | {"cx_Oracle"},
+    "oracle": sql_common | {"oracledb"},
     "postgres": sql_common | postgres_common,
     "presto": sql_common | pyhive_common | trino,
     # presto-on-hive is an alias for hive-metastore and needs to be kept in sync
@@ -443,14 +471,14 @@
     | pyhive_common
     | {"psycopg2-binary", "pymysql>=1.0.2"},
     "pulsar": {"requests"},
-    "redash": {"redash-toolbelt", "sql-metadata"} | sqllineage_lib,
+    "redash": {"redash-toolbelt", "sql-metadata"} | sqlglot_lib,
     "redshift": sql_common
     | redshift_common
     | usage_common
     | sqlglot_lib
     | classification_lib
     | {"db-dtypes"}  # Pandas extension data types
-    | {"cachetools"},
+    | cachetools_lib,
     "s3": {*s3_base, *data_lake_profiling},
     "gcs": {*s3_base, *data_lake_profiling},
     "abs": {*abs_base, *data_lake_profiling},
@@ -462,32 +490,38 @@
     "sqlalchemy": sql_common,
     "sql-queries": usage_common | sqlglot_lib,
     "slack": slack,
-    "superset": {
-        "requests",
-        "sqlalchemy",
-        "great_expectations",
-        "greenlet",
-    },
-    # FIXME: I don't think tableau uses sqllineage anymore so we should be able
-    # to remove that dependency.
-    "tableau": {"tableauserverclient>=0.24.0"} | sqllineage_lib | sqlglot_lib,
+    "superset": superset_common,
+    "preset": superset_common,
+    "tableau": {"tableauserverclient>=0.24.0"} | sqlglot_lib,
     "teradata": sql_common
     | usage_common
     | sqlglot_lib
-    | {"teradatasqlalchemy>=17.20.0.0"},
+    | {
+        # On 2024-10-30, teradatasqlalchemy 20.0.0.2 was released. This version seemed to cause issues
+        # in our CI, so we're pinning the version for now.
+        "teradatasqlalchemy>=17.20.0.0,<=20.0.0.2",
+    },
     "trino": sql_common | trino,
     "starburst-trino-usage": sql_common | usage_common | trino,
     "nifi": {"requests", "packaging", "requests-gssapi"},
-    "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"} | sqlglot_lib,
+    "powerbi": (
+        (
+            microsoft_common
+            | {"lark[regex]==1.1.4", "sqlparse", "more-itertools"}
+            | sqlglot_lib
+            | threading_timeout_common
+        )
+    ),
     "powerbi-report-server": powerbi_report_server,
     "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.2"},
-    "unity-catalog": databricks | sql_common | sqllineage_lib,
+    "unity-catalog": databricks | sql_common,
     # databricks is alias for unity-catalog and needs to be kept in sync
-    "databricks": databricks | sql_common | sqllineage_lib,
+    "databricks": databricks | sql_common,
     "fivetran": snowflake_common | bigquery_common | sqlglot_lib,
     "qlik-sense": sqlglot_lib | {"requests", "websocket-client"},
     "sigma": sqlglot_lib | {"requests"},
     "sac": sac,
+    "neo4j": {"pandas", "neo4j"},
 }
 
 # This is mainly used to exclude plugins from the Docker image.
@@ -536,7 +570,6 @@
     "types-pyOpenSSL",
     "types-click-spinner>=0.1.13.1",
     "types-ujson>=5.2.0",
-    "types-termcolor>=1.0.0",
     "types-Deprecated",
     "types-protobuf>=4.21.0.1",
     "sqlalchemy2-stubs",
@@ -545,6 +578,7 @@
 
 test_api_requirements = {
     "pytest>=6.2.2",
+    "pytest-timeout",
     # Missing numpy requirement in 8.0.0
     "deepdiff!=8.0.0",
     "PyYAML",
@@ -555,22 +589,26 @@
     "memray",
 }
 
-base_dev_requirements = {
-    *base_requirements,
-    *framework_common,
-    *mypy_stubs,
-    *s3_base,
+lint_requirements = {
     # This is pinned only to avoid spurious errors in CI.
     # We should make an effort to keep it up to date.
-    "black==22.12.0",
-    "coverage>=5.1",
-    "faker>=18.4.0",
+    "black==23.3.0",
     "flake8>=6.0.0",
     "flake8-tidy-imports>=4.3.0",
     "flake8-bugbear==23.3.12",
     "isort>=5.7.0",
     "mypy==1.10.1",
+}
+
+base_dev_requirements = {
+    *base_requirements,
+    *framework_common,
+    *mypy_stubs,
+    *s3_base,
+    *lint_requirements,
     *test_api_requirements,
+    "coverage>=5.1",
+    "faker>=18.4.0",
     "pytest-asyncio>=0.16.0",
     "pytest-cov>=2.8.1",
     "pytest-random-order~=1.1.0",
@@ -589,6 +627,7 @@
             "clickhouse-usage",
             "cockroachdb",
             "delta-lake",
+            "dremio",
             "druid",
             "elasticsearch",
             "feast",
@@ -630,6 +669,8 @@
             "qlik-sense",
             "sigma",
             "sac",
+            "cassandra",
+            "neo4j",
         ]
         if plugin
         for dependency in plugins[plugin]
@@ -687,6 +728,7 @@
         "s3 = datahub.ingestion.source.s3:S3Source",
         "dbt = datahub.ingestion.source.dbt.dbt_core:DBTCoreSource",
         "dbt-cloud = datahub.ingestion.source.dbt.dbt_cloud:DBTCloudSource",
+        "dremio = datahub.ingestion.source.dremio.dremio_source:DremioSource",
         "druid = datahub.ingestion.source.sql.druid:DruidSource",
         "dynamodb = datahub.ingestion.source.dynamodb.dynamodb:DynamoDBSource",
         "elasticsearch = datahub.ingestion.source.elastic_search:ElasticsearchSource",
@@ -698,8 +740,8 @@
         "hive = datahub.ingestion.source.sql.hive:HiveSource",
         "hive-metastore = datahub.ingestion.source.sql.hive_metastore:HiveMetastoreSource",
         "json-schema = datahub.ingestion.source.schema.json_schema:JsonSchemaSource",
-        "kafka = datahub.ingestion.source.kafka:KafkaSource",
-        "kafka-connect = datahub.ingestion.source.kafka_connect:KafkaConnectSource",
+        "kafka = datahub.ingestion.source.kafka.kafka:KafkaSource",
+        "kafka-connect = datahub.ingestion.source.kafka_connect.kafka_connect:KafkaConnectSource",
         "ldap = datahub.ingestion.source.ldap:LDAPSource",
         "looker = datahub.ingestion.source.looker.looker_source:LookerDashboardSource",
         "lookml = datahub.ingestion.source.looker.lookml_source:LookMLSource",
@@ -730,7 +772,7 @@
         "trino = datahub.ingestion.source.sql.trino:TrinoSource",
         "starburst-trino-usage = datahub.ingestion.source.usage.starburst_trino_usage:TrinoUsageSource",
         "nifi = datahub.ingestion.source.nifi:NifiSource",
-        "powerbi = datahub.ingestion.source.powerbi:PowerBiDashboardSource",
+        "powerbi = datahub.ingestion.source.powerbi.powerbi:PowerBiDashboardSource",
         "powerbi-report-server = datahub.ingestion.source.powerbi_report_server:PowerBiReportServerDashboardSource",
         "iceberg = datahub.ingestion.source.iceberg.iceberg:IcebergSource",
         "vertica = datahub.ingestion.source.sql.vertica:VerticaSource",
@@ -747,6 +789,8 @@
         "qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource",
         "sigma = datahub.ingestion.source.sigma.sigma:SigmaSource",
         "sac = datahub.ingestion.source.sac.sac:SACSource",
+        "cassandra = datahub.ingestion.source.cassandra.cassandra:CassandraSource",
+        "neo4j = datahub.ingestion.source.neo4j.neo4j_source:Neo4jSource",
     ],
     "datahub.ingestion.transformer.plugins": [
         "pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership",
@@ -888,6 +932,7 @@
         ),
         "cloud": ["acryl-datahub-cloud"],
         "dev": list(dev_requirements),
+        "lint": list(lint_requirements),
         "testing-utils": list(test_api_requirements),  # To import `datahub.testing`
         "integration-tests": list(full_test_dev_requirements),
         "debug": list(debug_requirements),
diff --git a/metadata-ingestion/sink_docs/metadata-file.md b/metadata-ingestion/sink_docs/metadata-file.md
index 7cac8d5542243..36c868828070e 100644
--- a/metadata-ingestion/sink_docs/metadata-file.md
+++ b/metadata-ingestion/sink_docs/metadata-file.md
@@ -34,4 +34,4 @@ Note that a `.` is used to denote nested fields in the YAML recipe.
 
 | Field    | Required | Default | Description               |
 | -------- | -------- | ------- | ------------------------- |
-| filename | ✅       |         | Path to file to write to. |
+| path     | ✅       |         | Path to file to write to. |
diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py b/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py
index 27317826264b8..0b04bfa4025a1 100644
--- a/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py
+++ b/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py
@@ -12,3 +12,10 @@
 )
 
 requests_logger.setLevel(logging.WARNING)
+
+__all__ = [
+    "AssertionCircuitBreaker",
+    "AssertionCircuitBreakerConfig",
+    "OperationCircuitBreaker",
+    "OperationCircuitBreakerConfig",
+]
diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py b/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py
index a3c54046faf68..7c1180536a90f 100644
--- a/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py
+++ b/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py
@@ -6,7 +6,7 @@
 from gql.transport.requests import RequestsHTTPTransport
 from pydantic import Field
 
-from datahub.configuration import ConfigModel
+from datahub.configuration.common import ConfigModel
 
 logger = logging.getLogger(__name__)
 
diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/__init__.py b/metadata-ingestion/src/datahub/api/entities/datajob/__init__.py
index 6d85a1569cb63..3a07300596822 100644
--- a/metadata-ingestion/src/datahub/api/entities/datajob/__init__.py
+++ b/metadata-ingestion/src/datahub/api/entities/datajob/__init__.py
@@ -1,2 +1,5 @@
 from datahub.api.entities.datajob.dataflow import DataFlow
 from datahub.api.entities.datajob.datajob import DataJob
+
+# TODO: Remove this and start importing directly from the inner files.
+__all__ = ["DataFlow", "DataJob"]
diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py b/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py
index f2436d56d5aca..e169c07445e96 100644
--- a/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py
+++ b/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py
@@ -3,7 +3,6 @@
 from typing import Callable, Dict, Iterable, List, Optional, Set, cast
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.source_common import ALL_ENV_TYPES
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -114,7 +113,7 @@ def generate_tags_aspect(self) -> List[GlobalTagsClass]:
 
     def _get_env(self) -> Optional[str]:
         env: Optional[str] = None
-        if self.env and self.env.upper() in ALL_ENV_TYPES:
+        if self.env and self.env.upper() in builder.ALL_ENV_TYPES:
             env = self.env.upper()
         else:
             logger.debug(
diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py
index 0f5d18c20e055..4958a68caa95f 100644
--- a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py
+++ b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py
@@ -3,7 +3,6 @@
 from typing import Callable, Dict, Iterable, List, Optional, Set
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.source_common import ALL_ENV_TYPES
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -109,7 +108,7 @@ def generate_mcp(
         self, materialize_iolets: bool = True
     ) -> Iterable[MetadataChangeProposalWrapper]:
         env: Optional[str] = None
-        if self.flow_urn.cluster.upper() in ALL_ENV_TYPES:
+        if self.flow_urn.cluster.upper() in builder.ALL_ENV_TYPES:
             env = self.flow_urn.cluster.upper()
         else:
             logger.debug(
diff --git a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
index 8f58fa469a7d9..2097922c15136 100644
--- a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
+++ b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
@@ -117,7 +117,7 @@ class DataProduct(ConfigModel):
     @pydantic.validator("assets", each_item=True)
     def assets_must_be_urns(cls, v: str) -> str:
         try:
-            Urn.create_from_string(v)
+            Urn.from_string(v)
         except Exception as e:
             raise ValueError(f"asset {v} is not an urn: {e}") from e
 
diff --git a/metadata-ingestion/src/datahub/api/entities/forms/forms.py b/metadata-ingestion/src/datahub/api/entities/forms/forms.py
index 9188ea33d6c68..fa6453851e0e7 100644
--- a/metadata-ingestion/src/datahub/api/entities/forms/forms.py
+++ b/metadata-ingestion/src/datahub/api/entities/forms/forms.py
@@ -16,8 +16,12 @@
 )
 from datahub.configuration.common import ConfigModel
 from datahub.emitter.mce_builder import (
+    make_container_urn,
     make_data_platform_urn,
+    make_domain_urn,
     make_group_urn,
+    make_tag_urn,
+    make_term_urn,
     make_user_urn,
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -36,6 +40,16 @@
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+FILTER_CRITERION_TYPES = "_entityType"
+FILTER_CRITERION_SUB_TYPES = "typeNames.keyword"
+FILTER_CRITERION_PLATFORMS = "platform.keyword"
+FILTER_CRITERION_PLATFORM_INSTANCES = "dataPlatformInstance.keyword"
+FILTER_CRITERION_DOMAINS = "domains.keyword"
+FILTER_CRITERION_CONTAINERS = "container.keyword"
+FILTER_CRITERION_OWNERS = "owners.keyword"
+FILTER_CRITERION_TAGS = "tags.keyword"
+FILTER_CRITERION_GLOSSARY_TERMS = "glossaryTerms.keyword"
+
 
 class PromptType(Enum):
     STRUCTURED_PROPERTY = "STRUCTURED_PROPERTY"
@@ -73,9 +87,14 @@ def has_value(cls, value):
 
 class Filters(ConfigModel):
     types: Optional[List[str]] = None
+    sub_types: Optional[List[str]] = None
     platforms: Optional[List[str]] = None
+    platform_instances: Optional[List[str]] = None
     domains: Optional[List[str]] = None
     containers: Optional[List[str]] = None
+    owners: Optional[List[str]] = None
+    tags: Optional[List[str]] = None
+    terms: Optional[List[str]] = None
 
 
 class Entities(ConfigModel):
@@ -105,7 +124,8 @@ class Forms(ConfigModel):
     @validator("urn", pre=True, always=True)
     def urn_must_be_present(cls, v, values):
         if not v:
-            assert values.get("id") is not None, "Form id must be present if urn is not"
+            if values.get("id") is None:
+                raise ValueError("Form id must be present if urn is not")
             return f"urn:li:form:{values['id']}"
         return v
 
@@ -249,29 +269,70 @@ def create_form_filters(self, emitter: DataHubGraph) -> Union[None, Exception]:
         # Loop through each entity and assign a filter for it
         if self.entities and self.entities.filters:
             filters = self.entities.filters
+
             if filters.types:
                 filters_raw.append(
-                    Forms.format_form_filter("_entityType", filters.types)
+                    Forms.format_form_filter(FILTER_CRITERION_TYPES, filters.types)
+                )
+
+            if filters.sub_types:
+                filters_raw.append(
+                    Forms.format_form_filter(
+                        FILTER_CRITERION_SUB_TYPES, filters.sub_types
+                    )
                 )
+
             if filters.platforms:
                 urns = [
                     make_data_platform_urn(platform) for platform in filters.platforms
                 ]
-                filters_raw.append(Forms.format_form_filter("platform", urns))
-            if filters.domains:
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_PLATFORMS, urns)
+                )
+
+            if filters.platform_instances:
                 urns = []
-                for domain in filters.domains:
-                    domain_urn = Forms.validate_domain_urn(domain)
-                    if domain_urn:
-                        urns.append(domain_urn)
-                filters_raw.append(Forms.format_form_filter("domains", urns))
+                for platform_instance in filters.platform_instances:
+                    platform_instance_urn = Forms.validate_platform_instance_urn(
+                        platform_instance
+                    )
+                    if platform_instance_urn:
+                        urns.append(platform_instance_urn)
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_PLATFORM_INSTANCES, urns)
+                )
+
+            if filters.domains:
+                urns = [make_domain_urn(domain) for domain in filters.domains]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_DOMAINS, urns)
+                )
+
             if filters.containers:
-                urns = []
-                for container in filters.containers:
-                    container_urn = Forms.validate_container_urn(container)
-                    if container_urn:
-                        urns.append(container_urn)
-                filters_raw.append(Forms.format_form_filter("container", urns))
+                urns = [
+                    make_container_urn(container) for container in filters.containers
+                ]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_CONTAINERS, urns)
+                )
+
+            if filters.owners:
+                urns = [make_user_urn(owner) for owner in filters.owners]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_OWNERS, urns)
+                )
+
+            if filters.tags:
+                urns = [make_tag_urn(tag) for tag in filters.tags]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_TAGS, urns)
+                )
+
+            if filters.terms:
+                urns = [make_term_urn(term) for term in filters.terms]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_GLOSSARY_TERMS, urns)
+                )
 
         filters_str = ", ".join(item for item in filters_raw)
         result = emitter.execute_graphql(
@@ -279,6 +340,7 @@ def create_form_filters(self, emitter: DataHubGraph) -> Union[None, Exception]:
                 form_urn=self.urn, filters=filters_str
             )
         )
+
         if not result:
             return Exception(
                 f"Could not bulk upload urns or filters for form {self.urn}."
@@ -314,25 +376,20 @@ def format_form_filter(field: str, urns: List[str]) -> str:
         return FIELD_FILTER_TEMPLATE.format(field=field, values=formatted_urns)
 
     @staticmethod
-    def validate_domain_urn(domain: str) -> Union[str, None]:
-        if domain.startswith("urn:li:domain:"):
-            return domain
+    def validate_platform_instance_urn(instance: str) -> Union[str, None]:
+        if instance.startswith("urn:li:dataPlatformInstance:"):
+            return instance
 
-        logger.warning(f"{domain} is not an urn. Unable to create domain filter.")
-        return None
-
-    @staticmethod
-    def validate_container_urn(container: str) -> Union[str, None]:
-        if container.startswith("urn:li:container:"):
-            return container
-
-        logger.warning(f"{container} is not an urn. Unable to create container filter.")
+        logger.warning(
+            f"{instance} is not an urn. Unable to create platform instance filter."
+        )
         return None
 
     @staticmethod
     def from_datahub(graph: DataHubGraph, urn: str) -> "Forms":
         form: Optional[FormInfoClass] = graph.get_aspect(urn, FormInfoClass)
-        assert form is not None
+        if form is None:
+            raise Exception("FormInfo aspect is None. Unable to create form.")
         prompts = []
         for prompt_raw in form.prompts:
             prompts.append(
diff --git a/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py b/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py
index 2b730ccb86f51..0ba43d7b101e5 100644
--- a/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py
+++ b/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py
@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, Iterable, List, Optional, Union
+from typing import Callable, Dict, Iterable, List, Optional, Tuple, Type, Union, cast
 
 from avrogen.dict_wrapper import DictWrapper
 from pydantic import BaseModel
@@ -14,7 +14,19 @@
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import DatahubKey
 from datahub.ingestion.graph.client import DataHubGraph
-from datahub.metadata.urns import PlatformResourceUrn
+from datahub.metadata.urns import (
+    DataPlatformInstanceUrn,
+    DataPlatformUrn,
+    PlatformResourceUrn,
+    Urn,
+)
+from datahub.utilities.openapi_utils import OpenAPIGraphClient
+from datahub.utilities.search_utils import (
+    ElasticDocumentQuery,
+    ElasticsearchQueryBuilder,
+    LogicalOperator,
+    SearchField,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -69,54 +81,60 @@ def to_resource_info(self) -> models.PlatformResourceInfoClass:
         )
 
 
-class OpenAPIGraphClient:
+class UrnSearchField(SearchField):
+    """
+    A search field that supports URN values.
+    TODO: Move this to search_utils after we make this more generic.
+    """
 
-    ENTITY_KEY_ASPECT_MAP = {
-        aspect_type.ASPECT_INFO.get("keyForEntity"): name
-        for name, aspect_type in models.ASPECT_NAME_MAP.items()
-        if aspect_type.ASPECT_INFO.get("keyForEntity")
-    }
+    def __init__(self, field_name: str, urn_value_extractor: Callable[[str], Urn]):
+        self.urn_value_extractor = urn_value_extractor
+        super().__init__(field_name)
 
-    def __init__(self, graph: DataHubGraph):
-        self.graph = graph
-        self.openapi_base = graph._gms_server.rstrip("/") + "/openapi/v3"
+    def get_search_value(self, value: str) -> str:
+        return str(self.urn_value_extractor(value))
 
-    def scroll_urns_by_filter(
-        self,
-        entity_type: str,
-        extra_or_filters: List[Dict[str, str]],
-    ) -> Iterable[str]:
-        """
-        Scroll through all urns that match the given filters
-        """
 
-        key_aspect = self.ENTITY_KEY_ASPECT_MAP.get(entity_type)
-        assert key_aspect, f"No key aspect found for entity type {entity_type}"
+class PlatformResourceSearchField(SearchField):
+    def __init__(self, field_name: str):
+        super().__init__(field_name)
 
-        count = 1000
-        query = " OR ".join(
-            [f"{filter['field']}:{filter['value']}" for filter in extra_or_filters]
+    @classmethod
+    def from_search_field(
+        cls, search_field: SearchField
+    ) -> "PlatformResourceSearchField":
+        # pretends to be a class method, but just returns the input
+        return search_field  # type: ignore
+
+
+class PlatformResourceSearchFields:
+    PRIMARY_KEY = PlatformResourceSearchField("primaryKey")
+    RESOURCE_TYPE = PlatformResourceSearchField("resourceType")
+    SECONDARY_KEYS = PlatformResourceSearchField("secondaryKeys")
+    PLATFORM = PlatformResourceSearchField.from_search_field(
+        UrnSearchField(
+            field_name="platform.keyword",
+            urn_value_extractor=DataPlatformUrn.create_from_id,
         )
-        scroll_id = None
-        while True:
-            response = self.graph._get_generic(
-                self.openapi_base + f"/entity/{entity_type.lower()}",
-                params={
-                    "systemMetadata": "false",
-                    "includeSoftDelete": "false",
-                    "skipCache": "false",
-                    "aspects": [key_aspect],
-                    "scrollId": scroll_id,
-                    "count": count,
-                    "query": query,
-                },
-            )
-            entities = response.get("entities", [])
-            scroll_id = response.get("scrollId")
-            for entity in entities:
-                yield entity["urn"]
-            if not scroll_id:
-                break
+    )
+    PLATFORM_INSTANCE = PlatformResourceSearchField.from_search_field(
+        UrnSearchField(
+            field_name="platformInstance.keyword",
+            urn_value_extractor=DataPlatformInstanceUrn.from_string,
+        )
+    )
+
+
+class ElasticPlatformResourceQuery(ElasticDocumentQuery[PlatformResourceSearchField]):
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def create_from(
+        cls: Type["ElasticPlatformResourceQuery"],
+        *args: Tuple[Union[str, PlatformResourceSearchField], str],
+    ) -> "ElasticPlatformResourceQuery":
+        return cast(ElasticPlatformResourceQuery, super().create_from(*args))
 
 
 class PlatformResource(BaseModel):
@@ -130,6 +148,12 @@ def remove(
         cls,
         key: PlatformResourceKey,
     ) -> "PlatformResource":
+        """
+        Creates a PlatformResource object with the removed status set to True.
+        Removed PlatformResource objects are used to soft-delete resources from
+        the graph.
+        To hard-delete a resource, use the delete method.
+        """
         return cls(
             id=key.id,
             removed=True,
@@ -186,10 +210,17 @@ def to_datahub(self, graph_client: DataHubGraph) -> None:
     def from_datahub(
         cls, graph_client: DataHubGraph, key: Union[PlatformResourceKey, str]
     ) -> Optional["PlatformResource"]:
+        """
+        Fetches a PlatformResource from the graph given a key.
+        Key can be either a PlatformResourceKey object or an urn string.
+        Returns None if the resource is not found.
+        """
         if isinstance(key, PlatformResourceKey):
             urn = PlatformResourceUrn(id=key.id)
         else:
             urn = PlatformResourceUrn.from_string(key)
+        if not graph_client.exists(str(urn)):
+            return None
         platform_resource = graph_client.get_entity_semityped(str(urn))
         return cls(
             id=urn.id,
@@ -216,28 +247,38 @@ def from_datahub(
 
     @staticmethod
     def search_by_key(
-        graph_client: DataHubGraph, key: str, primary: bool = True
+        graph_client: DataHubGraph,
+        key: str,
+        primary: bool = True,
+        is_exact: bool = True,
     ) -> Iterable["PlatformResource"]:
-        extra_or_filters = []
-        extra_or_filters.append(
-            {
-                "field": "primaryKey",
-                "condition": "EQUAL",
-                "value": key,
-            }
+        """
+        Searches for PlatformResource entities by primary or secondary key.
+
+        :param graph_client: DataHubGraph client
+        :param key: The key to search for
+        :param primary: Whether to search for primary only or expand the search
+            to secondary keys (default: True)
+        :param is_exact: Whether to search for an exact match (default: True)
+        :return: An iterable of PlatformResource objects
+        """
+
+        elastic_platform_resource_group = (
+            ElasticPlatformResourceQuery.create_from()
+            .group(LogicalOperator.OR)
+            .add_field_match(
+                PlatformResourceSearchFields.PRIMARY_KEY, key, is_exact=is_exact
+            )
         )
         if not primary:  # we expand the search to secondary keys
-            extra_or_filters.append(
-                {
-                    "field": "secondaryKeys",
-                    "condition": "EQUAL",
-                    "value": key,
-                }
+            elastic_platform_resource_group.add_field_match(
+                PlatformResourceSearchFields.SECONDARY_KEYS, key, is_exact=is_exact
             )
+        query = elastic_platform_resource_group.end()
         openapi_client = OpenAPIGraphClient(graph_client)
         for urn in openapi_client.scroll_urns_by_filter(
             entity_type="platformResource",
-            extra_or_filters=extra_or_filters,
+            query=query,
         ):
             platform_resource = PlatformResource.from_datahub(graph_client, urn)
             if platform_resource:
@@ -245,3 +286,21 @@ def search_by_key(
 
     def delete(self, graph_client: DataHubGraph, hard: bool = True) -> None:
         graph_client.delete_entity(str(PlatformResourceUrn(self.id)), hard=hard)
+
+    @staticmethod
+    def search_by_filters(
+        graph_client: DataHubGraph,
+        query: Union[
+            ElasticPlatformResourceQuery,
+            ElasticDocumentQuery,
+            ElasticsearchQueryBuilder,
+        ],
+    ) -> Iterable["PlatformResource"]:
+        openapi_client = OpenAPIGraphClient(graph_client)
+        for urn in openapi_client.scroll_urns_by_filter(
+            entity_type="platformResource",
+            query=query,
+        ):
+            platform_resource = PlatformResource.from_datahub(graph_client, urn)
+            if platform_resource:
+                yield platform_resource
diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py
index 5b188edf9563b..619f69b016262 100644
--- a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py
+++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py
@@ -9,12 +9,13 @@
 
 from datahub.configuration.common import ConfigModel
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
+from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import (
     PropertyValueClass,
     StructuredPropertyDefinitionClass,
 )
-from datahub.utilities.urns.urn import Urn
+from datahub.metadata.urns import DataTypeUrn, StructuredPropertyUrn, Urn
+from datahub.utilities.urns._urn_base import URN_TYPES
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -41,26 +42,28 @@ class AllowedValue(ConfigModel):
     description: Optional[str] = None
 
 
+VALID_ENTITY_TYPE_URNS = [
+    Urn.make_entity_type_urn(entity_type) for entity_type in URN_TYPES.keys()
+]
+_VALID_ENTITY_TYPES_STRING = f"Valid entity type urns are {', '.join(VALID_ENTITY_TYPE_URNS)}, etc... Ensure that the entity type is valid."
+
+
+def _validate_entity_type_urn(v: str) -> str:
+    urn = Urn.make_entity_type_urn(v)
+    if urn not in VALID_ENTITY_TYPE_URNS:
+        raise ValueError(
+            f"Input {v} is not a valid entity type urn. {_VALID_ENTITY_TYPES_STRING}"
+        )
+    v = str(urn)
+    return v
+
+
 class TypeQualifierAllowedTypes(ConfigModel):
     allowed_types: List[str]
 
-    @validator("allowed_types")
-    def validate_allowed_types(cls, v):
-        validated_entity_type_urns = []
-        if v:
-            with get_default_graph() as graph:
-                for et in v:
-                    validated_urn = Urn.make_entity_type_urn(et)
-                    if graph.exists(validated_urn):
-                        validated_entity_type_urns.append(validated_urn)
-                    else:
-                        logger.warn(
-                            f"Input {et} is not a valid entity type urn. Skipping."
-                        )
-        v = validated_entity_type_urns
-        if not v:
-            logger.warn("No allowed_types given within type_qualifier.")
-        return v
+    _check_allowed_types = validator("allowed_types", each_item=True, allow_reuse=True)(
+        _validate_entity_type_urn
+    )
 
 
 class StructuredProperties(ConfigModel):
@@ -77,84 +80,117 @@ class StructuredProperties(ConfigModel):
     type_qualifier: Optional[TypeQualifierAllowedTypes] = None
     immutable: Optional[bool] = False
 
+    _check_entity_types = validator("entity_types", each_item=True, allow_reuse=True)(
+        _validate_entity_type_urn
+    )
+
+    @validator("type")
+    def validate_type(cls, v: str) -> str:
+        # This logic is somewhat hacky, since we need to deal with
+        # 1. fully qualified urns
+        # 2. raw data types, that need to get the datahub namespace prefix
+        # While keeping the user-facing interface and error messages clean.
+
+        if not v.startswith("urn:li:") and not v.islower():
+            # Convert to lowercase if needed
+            v = v.lower()
+            logger.warning(
+                f"Structured property type should be lowercase. Updated to {v}"
+            )
+
+        urn = Urn.make_data_type_urn(v)
+
+        # Check if type is allowed
+        data_type_urn = DataTypeUrn.from_string(urn)
+        unqualified_data_type = data_type_urn.id
+        if unqualified_data_type.startswith("datahub."):
+            unqualified_data_type = unqualified_data_type[len("datahub.") :]
+        if not AllowedTypes.check_allowed_type(unqualified_data_type):
+            raise ValueError(
+                f"Type {unqualified_data_type} is not allowed. Allowed types are {AllowedTypes.values()}"
+            )
+
+        return urn
+
     @property
     def fqn(self) -> str:
         assert self.urn is not None
-        return (
-            self.qualified_name
-            or self.id
-            or Urn.create_from_string(self.urn).get_entity_id()[0]
-        )
+        id = StructuredPropertyUrn.from_string(self.urn).id
+        if self.qualified_name is not None:
+            # ensure that qualified name and ID match
+            assert (
+                self.qualified_name == id
+            ), "ID in the urn and the qualified_name must match"
+        return id
 
     @validator("urn", pre=True, always=True)
     def urn_must_be_present(cls, v, values):
         if not v:
-            assert "id" in values, "id must be present if urn is not"
+            if "id" not in values:
+                raise ValueError("id must be present if urn is not")
             return f"urn:li:structuredProperty:{values['id']}"
         return v
 
     @staticmethod
-    def create(file: str, graph: Optional[DataHubGraph] = None) -> None:
-        emitter: DataHubGraph = graph if graph else get_default_graph()
-
+    def from_yaml(file: str) -> List["StructuredProperties"]:
         with open(file) as fp:
             structuredproperties: List[dict] = yaml.safe_load(fp)
-            for structuredproperty_raw in structuredproperties:
-                structuredproperty = StructuredProperties.parse_obj(
-                    structuredproperty_raw
-                )
-                if not structuredproperty.type.islower():
-                    structuredproperty.type = structuredproperty.type.lower()
-                    logger.warn(
-                        f"Structured property type should be lowercase. Updated to {structuredproperty.type}"
-                    )
-                if not AllowedTypes.check_allowed_type(structuredproperty.type):
-                    raise ValueError(
-                        f"Type {structuredproperty.type} is not allowed. Allowed types are {AllowedTypes.values()}"
-                    )
-                mcp = MetadataChangeProposalWrapper(
-                    entityUrn=structuredproperty.urn,
-                    aspect=StructuredPropertyDefinitionClass(
-                        qualifiedName=structuredproperty.fqn,
-                        valueType=Urn.make_data_type_urn(structuredproperty.type),
-                        displayName=structuredproperty.display_name,
-                        description=structuredproperty.description,
-                        entityTypes=[
-                            Urn.make_entity_type_urn(entity_type)
-                            for entity_type in structuredproperty.entity_types or []
-                        ],
-                        cardinality=structuredproperty.cardinality,
-                        immutable=structuredproperty.immutable,
-                        allowedValues=(
-                            [
-                                PropertyValueClass(
-                                    value=v.value, description=v.description
-                                )
-                                for v in structuredproperty.allowed_values
-                            ]
-                            if structuredproperty.allowed_values
-                            else None
-                        ),
-                        typeQualifier=(
-                            {
-                                "allowedTypes": structuredproperty.type_qualifier.allowed_types
-                            }
-                            if structuredproperty.type_qualifier
-                            else None
-                        ),
-                    ),
-                )
-                emitter.emit_mcp(mcp)
-
-                logger.info(f"Created structured property {structuredproperty.urn}")
+
+        result: List[StructuredProperties] = []
+        for structuredproperty_raw in structuredproperties:
+            result.append(StructuredProperties.parse_obj(structuredproperty_raw))
+        return result
+
+    def generate_mcps(self) -> List[MetadataChangeProposalWrapper]:
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=self.urn,
+            aspect=StructuredPropertyDefinitionClass(
+                qualifiedName=self.fqn,
+                valueType=Urn.make_data_type_urn(self.type),
+                displayName=self.display_name,
+                description=self.description,
+                entityTypes=[
+                    Urn.make_entity_type_urn(entity_type)
+                    for entity_type in self.entity_types or []
+                ],
+                cardinality=self.cardinality,
+                immutable=self.immutable,
+                allowedValues=(
+                    [
+                        PropertyValueClass(value=v.value, description=v.description)
+                        for v in self.allowed_values
+                    ]
+                    if self.allowed_values
+                    else None
+                ),
+                typeQualifier=(
+                    {"allowedTypes": self.type_qualifier.allowed_types}
+                    if self.type_qualifier
+                    else None
+                ),
+            ),
+        )
+        return [mcp]
+
+    @staticmethod
+    def create(file: str, graph: DataHubGraph) -> None:
+        # TODO: Deprecate this method.
+        structuredproperties = StructuredProperties.from_yaml(file)
+        for structuredproperty in structuredproperties:
+            for mcp in structuredproperty.generate_mcps():
+                graph.emit_mcp(mcp)
+
+            logger.info(f"Created structured property {structuredproperty.urn}")
 
     @classmethod
     def from_datahub(cls, graph: DataHubGraph, urn: str) -> "StructuredProperties":
-
         structured_property: Optional[
             StructuredPropertyDefinitionClass
         ] = graph.get_aspect(urn, StructuredPropertyDefinitionClass)
-        assert structured_property is not None
+        if structured_property is None:
+            raise Exception(
+                "StructuredPropertyDefinition aspect is None. Unable to create structured property."
+            )
         return StructuredProperties(
             urn=urn,
             qualified_name=structured_property.qualifiedName,
diff --git a/metadata-ingestion/src/datahub/api/graphql/__init__.py b/metadata-ingestion/src/datahub/api/graphql/__init__.py
index e8c8d22bbb93d..d818b19092fcb 100644
--- a/metadata-ingestion/src/datahub/api/graphql/__init__.py
+++ b/metadata-ingestion/src/datahub/api/graphql/__init__.py
@@ -1,2 +1,4 @@
 from datahub.api.graphql.assertion import Assertion
 from datahub.api.graphql.operation import Operation
+
+__all__ = ["Assertion", "Operation"]
diff --git a/metadata-ingestion/src/datahub/cli/check_cli.py b/metadata-ingestion/src/datahub/cli/check_cli.py
index 6e9bfddd350f9..fbe07b64f0e15 100644
--- a/metadata-ingestion/src/datahub/cli/check_cli.py
+++ b/metadata-ingestion/src/datahub/cli/check_cli.py
@@ -188,9 +188,13 @@ def sql_format(sql: str, platform: str) -> None:
 @click.option(
     "--sql",
     type=str,
-    required=True,
     help="The SQL query to parse",
 )
+@click.option(
+    "--sql-file",
+    type=click.Path(exists=True, dir_okay=False, readable=True),
+    help="The SQL file to parse",
+)
 @click.option(
     "--platform",
     type=str,
@@ -218,25 +222,44 @@ def sql_format(sql: str, platform: str) -> None:
     type=str,
     help="The default schema to use for unqualified table names",
 )
+@click.option(
+    "--online/--offline",
+    type=bool,
+    is_flag=True,
+    default=True,
+    help="Run in offline mode and disable schema-aware parsing.",
+)
 @telemetry.with_telemetry()
 def sql_lineage(
-    sql: str,
+    sql: Optional[str],
+    sql_file: Optional[str],
     platform: str,
     default_db: Optional[str],
     default_schema: Optional[str],
     platform_instance: Optional[str],
     env: str,
+    online: bool,
 ) -> None:
     """Parse the lineage of a SQL query.
 
-    This performs schema-aware parsing in order to generate column-level lineage.
-    If the relevant tables are not in DataHub, this will be less accurate.
+    In online mode (the default), we perform schema-aware parsing in order to generate column-level lineage.
+    If offline mode is enabled or if the relevant tables are not in DataHub, this will be less accurate.
     """
 
-    graph = get_default_graph()
+    from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
+
+    if sql is None:
+        if sql_file is None:
+            raise click.UsageError("Either --sql or --sql-file must be provided")
+        sql = pathlib.Path(sql_file).read_text()
+
+    graph = None
+    if online:
+        graph = get_default_graph()
 
-    lineage = graph.parse_sql_lineage(
+    lineage = create_lineage_sql_parsed_result(
         sql,
+        graph=graph,
         platform=platform,
         platform_instance=platform_instance,
         env=env,
@@ -245,7 +268,9 @@ def sql_lineage(
     )
 
     logger.debug("Sql parsing debug info: %s", lineage.debug_info)
-    if lineage.debug_info.error:
+    if lineage.debug_info.table_error:
+        raise lineage.debug_info.table_error
+    elif lineage.debug_info.error:
         logger.debug("Sql parsing error details", exc_info=lineage.debug_info.error)
 
     click.echo(lineage.json(indent=4))
diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py
index 1b9cccb1cbc21..f80181192ba58 100644
--- a/metadata-ingestion/src/datahub/cli/cli_utils.py
+++ b/metadata-ingestion/src/datahub/cli/cli_utils.py
@@ -327,6 +327,8 @@ def _ensure_valid_gms_url_acryl_cloud(url: str) -> str:
         url = f"{url}/gms"
     elif url.endswith("acryl.io/"):
         url = f"{url}gms"
+    if url.endswith("acryl.io/api/gms"):
+        url = url.replace("acryl.io/api/gms", "acryl.io/gms")
 
     return url
 
diff --git a/metadata-ingestion/src/datahub/cli/delete_cli.py b/metadata-ingestion/src/datahub/cli/delete_cli.py
index 584bfc1f7cda5..1a75459a92c5c 100644
--- a/metadata-ingestion/src/datahub/cli/delete_cli.py
+++ b/metadata-ingestion/src/datahub/cli/delete_cli.py
@@ -1,4 +1,5 @@
 import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
 from datetime import datetime
 from random import choices
@@ -213,6 +214,50 @@ def references(urn: str, dry_run: bool, force: bool) -> None:
         logger.info(f"Deleted {references_count} references to {urn}")
 
 
+@delete.command()
+@click.option("--urn", required=False, type=str, help="the urn of the entity")
+@click.option(
+    "-p",
+    "--platform",
+    required=False,
+    type=str,
+    help="Platform filter (e.g. snowflake)",
+)
+@click.option(
+    "-b",
+    "--batch-size",
+    required=False,
+    default=3000,
+    type=int,
+    help="Batch size when querying for entities to un-soft delete."
+    "Maximum 10000. Large batch sizes may cause timeouts.",
+)
+def undo_by_filter(
+    urn: Optional[str], platform: Optional[str], batch_size: int
+) -> None:
+    """
+    Undo soft deletion by filters
+    """
+    graph = get_default_graph()
+    logger.info(f"Using {graph}")
+    if urn:
+        graph.set_soft_delete_status(urn=urn, delete=False)
+    else:
+        urns = list(
+            graph.get_urns_by_filter(
+                platform=platform,
+                query="*",
+                status=RemovedStatusFilter.ONLY_SOFT_DELETED,
+                batch_size=batch_size,
+            )
+        )
+        logger.info(f"Going to un-soft delete {len(urns)} urns")
+        urns_iter = progressbar.progressbar(urns, redirect_stdout=True)
+        for urn in urns_iter:
+            assert urn
+            graph.set_soft_delete_status(urn=urn, delete=False)
+
+
 @delete.command(no_args_is_help=True)
 @click.option(
     "--urn",
@@ -301,6 +346,9 @@ def references(urn: str, dry_run: bool, force: bool) -> None:
     default=False,
     help="Only delete soft-deleted entities, for hard deletion",
 )
+@click.option(
+    "--workers", type=int, default=1, help="Num of workers to use for deletion."
+)
 @upgrade.check_upgrade
 @telemetry.with_telemetry()
 def by_filter(
@@ -318,6 +366,7 @@ def by_filter(
     batch_size: int,
     dry_run: bool,
     only_soft_deleted: bool,
+    workers: int = 1,
 ) -> None:
     """Delete metadata from datahub using a single urn or a combination of filters."""
 
@@ -338,16 +387,19 @@ def by_filter(
     # TODO: add some validation on entity_type
 
     if not force and not soft and not dry_run:
+        message = (
+            "Hard deletion will permanently delete data from DataHub and can be slow. "
+            "We generally recommend using soft deletes instead. "
+            "Do you want to continue?"
+        )
         if only_soft_deleted:
             click.confirm(
-                "This will permanently delete data from DataHub. Do you want to continue?",
+                message,
                 abort=True,
             )
         else:
             click.confirm(
-                "Hard deletion will permanently delete data from DataHub and can be slow. "
-                "We generally recommend using soft deletes instead. "
-                "Do you want to continue?",
+                message,
                 abort=True,
             )
 
@@ -418,26 +470,64 @@ def by_filter(
                 abort=True,
             )
 
-    urns_iter = urns
-    if not delete_by_urn and not dry_run:
-        urns_iter = progressbar.progressbar(urns, redirect_stdout=True)
+    _delete_urns_parallel(
+        graph=graph,
+        urns=urns,
+        aspect_name=aspect,
+        soft=soft,
+        dry_run=dry_run,
+        delete_by_urn=delete_by_urn,
+        start_time=start_time,
+        end_time=end_time,
+        workers=workers,
+    )
+
 
-    # Run the deletion.
+def _delete_urns_parallel(
+    graph: DataHubGraph,
+    urns: List[str],
+    delete_by_urn: bool,
+    start_time: Optional[datetime],
+    end_time: Optional[datetime],
+    aspect_name: Optional[str] = None,
+    soft: bool = True,
+    dry_run: bool = False,
+    workers: int = 1,
+) -> None:
     deletion_result = DeletionResult()
-    with PerfTimer() as timer:
-        for urn in urns_iter:
-            one_result = _delete_one_urn(
-                graph=graph,
-                urn=urn,
-                aspect_name=aspect,
-                soft=soft,
-                dry_run=dry_run,
-                start_time=start_time,
-                end_time=end_time,
+
+    def process_urn(urn):
+        return _delete_one_urn(
+            graph=graph,
+            urn=urn,
+            aspect_name=aspect_name,
+            soft=soft,
+            dry_run=dry_run,
+            start_time=start_time,
+            end_time=end_time,
+        )
+
+    with PerfTimer() as timer, ThreadPoolExecutor(max_workers=workers) as executor:
+        future_to_urn = {executor.submit(process_urn, urn): urn for urn in urns}
+
+        completed_futures = as_completed(future_to_urn)
+        if not delete_by_urn and not dry_run:
+            futures_iter = progressbar.progressbar(
+                as_completed(future_to_urn),
+                max_value=len(future_to_urn),
+                redirect_stdout=True,
             )
-            deletion_result.merge(one_result)
+        else:
+            futures_iter = completed_futures
+
+        for future in futures_iter:
+            try:
+                one_result = future.result()
+                deletion_result.merge(one_result)
+            except Exception as e:
+                urn = future_to_urn[future]
+                click.secho(f"Error processing URN {urn}: {e}", fg="red")
 
-    # Report out a summary of the deletion result.
     click.echo(
         deletion_result.format_message(
             dry_run=dry_run, soft=soft, time_sec=timer.elapsed_seconds()
diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py
index 8654e6c282ba8..fcab07a1c2aaf 100644
--- a/metadata-ingestion/src/datahub/cli/ingest_cli.py
+++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py
@@ -1,4 +1,3 @@
-import asyncio
 import csv
 import json
 import logging
@@ -24,9 +23,11 @@
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
+from datahub.utilities.perf_timer import PerfTimer
 
 logger = logging.getLogger(__name__)
 
+INGEST_SRC_TABLE_COLUMNS = ["runId", "source", "startTime", "status", "URN"]
 RUNS_TABLE_COLUMNS = ["runId", "rows", "created at"]
 RUN_TABLE_COLUMNS = ["urn", "aspect name", "created at"]
 
@@ -126,7 +127,7 @@ def run(
 ) -> None:
     """Ingest metadata into DataHub."""
 
-    async def run_pipeline_to_completion(pipeline: Pipeline) -> int:
+    def run_pipeline_to_completion(pipeline: Pipeline) -> int:
         logger.info("Starting metadata ingestion")
         with click_spinner.spinner(disable=no_spinner or no_progress):
             try:
@@ -166,44 +167,25 @@ async def run_pipeline_to_completion(pipeline: Pipeline) -> int:
         # The default is "datahub" reporting. The extra flag will disable it.
         report_to = None
 
-    async def run_ingestion_and_check_upgrade() -> int:
-        # TRICKY: We want to make sure that the Pipeline.create() call happens on the
-        # same thread as the rest of the ingestion. As such, we must initialize the
-        # pipeline inside the async function so that it happens on the same event
-        # loop, and hence the same thread.
-
-        # logger.debug(f"Using config: {pipeline_config}")
-        pipeline = Pipeline.create(
-            pipeline_config,
-            dry_run=dry_run,
-            preview_mode=preview,
-            preview_workunits=preview_workunits,
-            report_to=report_to,
-            no_progress=no_progress,
-            raw_config=raw_pipeline_config,
-        )
+    # logger.debug(f"Using config: {pipeline_config}")
+    pipeline = Pipeline.create(
+        pipeline_config,
+        dry_run=dry_run,
+        preview_mode=preview,
+        preview_workunits=preview_workunits,
+        report_to=report_to,
+        no_progress=no_progress,
+        raw_config=raw_pipeline_config,
+    )
+    with PerfTimer() as timer:
+        ret = run_pipeline_to_completion(pipeline)
 
-        version_stats_future = asyncio.ensure_future(
-            upgrade.retrieve_version_stats(pipeline.ctx.graph)
+    # The main ingestion has completed. If it was successful, potentially show an upgrade nudge message.
+    if ret == 0:
+        upgrade.check_upgrade_post(
+            main_method_runtime=timer.elapsed_seconds(), graph=pipeline.ctx.graph
         )
-        ingestion_future = asyncio.ensure_future(run_pipeline_to_completion(pipeline))
-        ret = await ingestion_future
-
-        # The main ingestion has completed. If it was successful, potentially show an upgrade nudge message.
-        if ret == 0:
-            try:
-                # we check the other futures quickly on success
-                version_stats = await asyncio.wait_for(version_stats_future, 0.5)
-                upgrade.maybe_print_upgrade_message(version_stats=version_stats)
-            except Exception as e:
-                logger.debug(
-                    f"timed out with {e} waiting for version stats to be computed... skipping ahead."
-                )
-
-        return ret
 
-    loop = asyncio.get_event_loop()
-    ret = loop.run_until_complete(run_ingestion_and_check_upgrade())
     if ret:
         sys.exit(ret)
     # don't raise SystemExit if there's no error
@@ -456,6 +438,115 @@ def mcps(path: str) -> None:
     sys.exit(ret)
 
 
+@ingest.command()
+@click.argument("page_offset", type=int, default=0)
+@click.argument("page_size", type=int, default=100)
+@click.option("--urn", type=str, default=None, help="Filter by ingestion source URN.")
+@click.option(
+    "--source", type=str, default=None, help="Filter by ingestion source name."
+)
+@upgrade.check_upgrade
+@telemetry.with_telemetry()
+def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) -> None:
+    """List ingestion source runs with their details, optionally filtered by URN or source."""
+
+    query = """
+    query listIngestionRuns($input: ListIngestionSourcesInput!) {
+      listIngestionSources(input: $input) {
+        ingestionSources {
+          urn
+          name
+          executions {
+            executionRequests {
+              id
+              result {
+                startTimeMs
+                status
+              }
+            }
+          }
+        }
+      }
+    }
+    """
+
+    # filter by urn and/or source using CONTAINS
+    filters = []
+    if urn:
+        filters.append({"field": "urn", "values": [urn], "condition": "CONTAIN"})
+    if source:
+        filters.append({"field": "name", "values": [source], "condition": "CONTAIN"})
+
+    variables = {
+        "input": {
+            "start": page_offset,
+            "count": page_size,
+            "filters": filters,
+        }
+    }
+
+    client = get_default_graph()
+    session = client._session
+    gms_host = client.config.server
+
+    url = f"{gms_host}/api/graphql"
+    try:
+        response = session.post(url, json={"query": query, "variables": variables})
+        response.raise_for_status()
+    except Exception as e:
+        click.echo(f"Error fetching data: {str(e)}")
+        return
+
+    try:
+        data = response.json()
+    except ValueError:
+        click.echo("Failed to parse JSON response from server.")
+        return
+
+    if not data:
+        click.echo("No response received from the server.")
+        return
+
+    # when urn or source filter does not match, exit gracefully
+    if (
+        not isinstance(data.get("data"), dict)
+        or "listIngestionSources" not in data["data"]
+    ):
+        click.echo("No matching ingestion sources found. Please check your filters.")
+        return
+
+    ingestion_sources = data["data"]["listIngestionSources"]["ingestionSources"]
+    if not ingestion_sources:
+        click.echo("No ingestion sources or executions found.")
+        return
+
+    rows = []
+    for ingestion_source in ingestion_sources:
+        urn = ingestion_source.get("urn", "N/A")
+        name = ingestion_source.get("name", "N/A")
+
+        executions = ingestion_source.get("executions", {}).get("executionRequests", [])
+        for execution in executions:
+            execution_id = execution.get("id", "N/A")
+            start_time = execution.get("result", {}).get("startTimeMs", "N/A")
+            start_time = (
+                datetime.fromtimestamp(start_time / 1000).strftime("%Y-%m-%d %H:%M:%S")
+                if start_time != "N/A"
+                else "N/A"
+            )
+            status = execution.get("result", {}).get("status", "N/A")
+
+            rows.append([execution_id, name, start_time, status, urn])
+
+    click.echo(
+        tabulate(
+            rows,
+            headers=INGEST_SRC_TABLE_COLUMNS,
+            tablefmt="grid",
+        )
+    )
+
+
 @ingest.command()
 @click.argument("page_offset", type=int, default=0)
 @click.argument("page_size", type=int, default=100)
diff --git a/metadata-ingestion/src/datahub/cli/json_file.py b/metadata-ingestion/src/datahub/cli/json_file.py
index 8fb3ea46c64e8..c2c17d92e5116 100644
--- a/metadata-ingestion/src/datahub/cli/json_file.py
+++ b/metadata-ingestion/src/datahub/cli/json_file.py
@@ -1,12 +1,15 @@
 import logging
 
+from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.file import GenericFileSource
 
 logger = logging.getLogger(__name__)
 
 
 def check_mce_file(filepath: str) -> str:
-    mce_source = GenericFileSource.create({"filename": filepath}, None)
+    mce_source = GenericFileSource.create(
+        {"filename": filepath}, PipelineContext(run_id="json-file")
+    )
     for _ in mce_source.get_workunits():
         pass
     if len(mce_source.get_report().failures):
diff --git a/metadata-ingestion/src/datahub/cli/put_cli.py b/metadata-ingestion/src/datahub/cli/put_cli.py
index 989b1a6d02fd0..d3a6fb5caaf19 100644
--- a/metadata-ingestion/src/datahub/cli/put_cli.py
+++ b/metadata-ingestion/src/datahub/cli/put_cli.py
@@ -6,11 +6,12 @@
 
 from datahub.cli.cli_utils import post_entity
 from datahub.configuration.config_loader import load_config_file
-from datahub.emitter.mcp import MetadataChangeProposalWrapper, SystemMetadataClass
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import get_default_graph
 from datahub.metadata.schema_classes import (
     DataPlatformInfoClass as DataPlatformInfo,
     PlatformTypeClass,
+    SystemMetadataClass,
 )
 from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
@@ -104,7 +105,7 @@ def platform(
     """
 
     if name.startswith(f"urn:li:{DataPlatformUrn.ENTITY_TYPE}"):
-        platform_urn = DataPlatformUrn.create_from_string(name)
+        platform_urn = DataPlatformUrn.from_string(name)
         platform_name = platform_urn.get_entity_id_as_string()
     else:
         platform_name = name.lower()
diff --git a/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py b/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py
index afac38e29722e..857a6fbb4e18e 100644
--- a/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py
+++ b/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py
@@ -45,7 +45,7 @@ def _get_owner_urn(maybe_urn: str) -> str:
 
 def _abort_if_non_existent_urn(graph: DataHubGraph, urn: str, operation: str) -> None:
     try:
-        parsed_urn: Urn = Urn.create_from_string(urn)
+        parsed_urn: Urn = Urn.from_string(urn)
         entity_type = parsed_urn.get_type()
     except Exception:
         click.secho(f"Provided urn {urn} does not seem valid", fg="red")
@@ -363,7 +363,7 @@ def remove_owner(urn: str, owner_urn: str) -> None:
     with get_default_graph() as graph:
         _abort_if_non_existent_urn(graph, urn, "remove owners")
         for mcp in dataproduct_patcher.build():
-            print(json.dumps(mcp.to_obj()))
+            click.echo(json.dumps(mcp.to_obj()))
             graph.emit(mcp)
 
 
diff --git a/metadata-ingestion/src/datahub/cli/specific/structuredproperties_cli.py b/metadata-ingestion/src/datahub/cli/specific/structuredproperties_cli.py
index 4162d44b9b0ea..42285cf13a5dd 100644
--- a/metadata-ingestion/src/datahub/cli/specific/structuredproperties_cli.py
+++ b/metadata-ingestion/src/datahub/cli/specific/structuredproperties_cli.py
@@ -31,7 +31,8 @@ def properties() -> None:
 def upsert(file: Path) -> None:
     """Upsert structured properties in DataHub."""
 
-    StructuredProperties.create(str(file))
+    with get_default_graph() as graph:
+        StructuredProperties.create(str(file), graph)
 
 
 @properties.command(
diff --git a/metadata-ingestion/src/datahub/cli/timeline_cli.py b/metadata-ingestion/src/datahub/cli/timeline_cli.py
index 08672528abb5d..37089e6f051f0 100644
--- a/metadata-ingestion/src/datahub/cli/timeline_cli.py
+++ b/metadata-ingestion/src/datahub/cli/timeline_cli.py
@@ -6,7 +6,6 @@
 
 import click
 from requests import Response
-from termcolor import colored
 
 from datahub.emitter.mce_builder import dataset_urn_to_key, schema_field_urn_to_key
 from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
@@ -44,14 +43,14 @@ def pretty_id(id: Optional[str]) -> str:
             assert schema_field_key is not None
             field_path = schema_field_key.fieldPath
 
-            return f"{colored('field','cyan')}:{colored(pretty_field_path(field_path),'white')}"
+            return f"{click.style('field', fg='cyan')}:{click.style(pretty_field_path(field_path), fg='white')}"
     if id.startswith("[version=2.0]"):
-        return f"{colored('field','cyan')}:{colored(pretty_field_path(id),'white')}"
+        return f"{click.style('field', fg='cyan')}:{click.style(pretty_field_path(id), fg='white')}"
 
     if id.startswith("urn:li:dataset"):
         dataset_key = dataset_urn_to_key(id)
         if dataset_key:
-            return f"{colored('dataset','cyan')}:{colored(dataset_key.platform[len('urn:li:dataPlatform:'):],'white')}:{colored(dataset_key.name,'white')}"
+            return f"{click.style('dataset', fg='cyan')}:{click.style(dataset_key.platform[len('urn:li:dataPlatform:'):], fg='white')}:{click.style(dataset_key.name, fg='white')}"
     # failed to prettify, return original
     return id
 
@@ -196,8 +195,8 @@ def timeline(
                 else "red"
             )
 
-            print(
-                f"{colored(change_instant,'cyan')} - {colored(change_txn['semVer'],change_color)}"
+            click.echo(
+                f"{click.style(change_instant, fg='cyan')} - {click.style(change_txn['semVer'], fg=change_color)}"
             )
             if change_txn["changeEvents"] is not None:
                 for change_event in change_txn["changeEvents"]:
@@ -216,8 +215,8 @@ def timeline(
                         or change_event.get("entityUrn")
                         or ""
                     )
-                    print(
-                        f"\t{colored(change_event.get('changeType') or change_event.get('operation'),event_change_color)} {change_event.get('category')} {target_string} {element_string}: {change_event['description']}"
+                    click.echo(
+                        f"\t{click.style(change_event.get('changeType') or change_event.get('operation'), fg=event_change_color)} {change_event.get('category')} {target_string} {element_string}: {change_event['description']}"
                     )
     else:
         click.echo(
diff --git a/metadata-ingestion/src/datahub/configuration/__init__.py b/metadata-ingestion/src/datahub/configuration/__init__.py
index 008d788072d0a..21979829a4453 100644
--- a/metadata-ingestion/src/datahub/configuration/__init__.py
+++ b/metadata-ingestion/src/datahub/configuration/__init__.py
@@ -1,5 +1,4 @@
 from datahub.configuration.common import (
-    ConfigModel,
-    ConfigurationMechanism,
-    DynamicTypedConfig,
+    ConfigModel as ConfigModel,
+    DynamicTypedConfig as DynamicTypedConfig,
 )
diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py
index 0ce7127b44053..7df007e087979 100644
--- a/metadata-ingestion/src/datahub/configuration/common.py
+++ b/metadata-ingestion/src/datahub/configuration/common.py
@@ -21,7 +21,7 @@
 from pydantic.fields import Field
 from typing_extensions import Protocol
 
-from datahub.configuration._config_enum import ConfigEnum
+from datahub.configuration._config_enum import ConfigEnum as ConfigEnum  # noqa: I250
 from datahub.configuration.pydantic_migration_helpers import PYDANTIC_VERSION_2
 from datahub.utilities.dedup_list import deduplicate_list
 
@@ -258,7 +258,7 @@ def allow_all(cls) -> "AllowDenyPattern":
         return AllowDenyPattern()
 
     def allowed(self, string: str) -> bool:
-        if self._denied(string):
+        if self.denied(string):
             return False
 
         return any(
@@ -266,7 +266,7 @@ def allowed(self, string: str) -> bool:
             for allow_pattern in self.allow
         )
 
-    def _denied(self, string: str) -> bool:
+    def denied(self, string: str) -> bool:
         for deny_pattern in self.deny:
             if re.match(deny_pattern, string, self.regex_flags):
                 return True
@@ -290,7 +290,7 @@ def get_allowed_list(self) -> List[str]:
             raise ValueError(
                 "allow list must be fully specified to get list of allowed strings"
             )
-        return [a for a in self.allow if not self._denied(a)]
+        return [a for a in self.allow if not self.denied(a)]
 
     def __eq__(self, other):  # type: ignore
         return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
diff --git a/metadata-ingestion/src/datahub/configuration/git.py b/metadata-ingestion/src/datahub/configuration/git.py
index d237cd9ddd306..e7e9bfd43adca 100644
--- a/metadata-ingestion/src/datahub/configuration/git.py
+++ b/metadata-ingestion/src/datahub/configuration/git.py
@@ -24,7 +24,11 @@ class GitReference(ConfigModel):
         "main",
         description="Branch on which your files live by default. Typically main or master. This can also be a commit hash.",
     )
-
+    url_subdir: Optional[str] = Field(
+        default=None,
+        description="Prefix to prepend when generating URLs for files - useful when files are in a subdirectory. "
+        "Only affects URL generation, not git operations.",
+    )
     url_template: Optional[str] = Field(
         None,
         description=f"Template for generating a URL to a file in the repo e.g. '{_GITHUB_URL_TEMPLATE}'. We can infer this for GitHub and GitLab repos, and it is otherwise required."
@@ -68,6 +72,8 @@ def infer_url_template(cls, url_template: Optional[str], values: dict) -> str:
 
     def get_url_for_file_path(self, file_path: str) -> str:
         assert self.url_template
+        if self.url_subdir:
+            file_path = f"{self.url_subdir}/{file_path}"
         return self.url_template.format(
             repo_url=self.repo, branch=self.branch, file_path=file_path
         )
diff --git a/metadata-ingestion/src/datahub/configuration/json_loader.py b/metadata-ingestion/src/datahub/configuration/json_loader.py
index 35667eb5951fc..6ecb741be528d 100644
--- a/metadata-ingestion/src/datahub/configuration/json_loader.py
+++ b/metadata-ingestion/src/datahub/configuration/json_loader.py
@@ -1,7 +1,7 @@
 import json
 from typing import IO
 
-from datahub.configuration import ConfigurationMechanism
+from datahub.configuration.common import ConfigurationMechanism
 
 
 class JsonConfigurationMechanism(ConfigurationMechanism):
diff --git a/metadata-ingestion/src/datahub/configuration/kafka.py b/metadata-ingestion/src/datahub/configuration/kafka.py
index 07e2f759bb3ff..b8d9ff994a51a 100644
--- a/metadata-ingestion/src/datahub/configuration/kafka.py
+++ b/metadata-ingestion/src/datahub/configuration/kafka.py
@@ -1,6 +1,7 @@
 from pydantic import Field, validator
 
-from datahub.configuration.common import ConfigModel
+from datahub.configuration.common import ConfigModel, ConfigurationError
+from datahub.configuration.kafka_consumer_config import CallableConsumerConfig
 from datahub.configuration.validate_host_port import validate_host_port
 
 
@@ -36,6 +37,16 @@ class KafkaConsumerConnectionConfig(_KafkaConnectionConfig):
         description="Extra consumer config serialized as JSON. These options will be passed into Kafka's DeserializingConsumer. See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#deserializingconsumer and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md .",
     )
 
+    @validator("consumer_config")
+    @classmethod
+    def resolve_callback(cls, value: dict) -> dict:
+        if CallableConsumerConfig.is_callable_config(value):
+            try:
+                value = CallableConsumerConfig(value).callable_config()
+            except Exception as e:
+                raise ConfigurationError(e)
+        return value
+
 
 class KafkaProducerConnectionConfig(_KafkaConnectionConfig):
     """Configuration class for holding connectivity information for Kafka producers"""
diff --git a/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py
new file mode 100644
index 0000000000000..f08c78cadc0b2
--- /dev/null
+++ b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py
@@ -0,0 +1,68 @@
+import inspect
+import logging
+from typing import Any, Dict, Optional
+
+from datahub.ingestion.api.registry import import_path
+
+logger = logging.getLogger(__name__)
+
+
+class CallableConsumerConfig:
+    CALLBACK_ATTRIBUTE: str = "oauth_cb"
+
+    def __init__(self, config: Dict[str, Any]):
+        self._config = config
+
+        self._resolve_oauth_callback()
+
+    def callable_config(self) -> Dict[str, Any]:
+        return self._config
+
+    @staticmethod
+    def is_callable_config(config: Dict[str, Any]) -> bool:
+        return CallableConsumerConfig.CALLBACK_ATTRIBUTE in config
+
+    def get_call_back_attribute(self) -> Optional[str]:
+        return self._config.get(CallableConsumerConfig.CALLBACK_ATTRIBUTE)
+
+    def _resolve_oauth_callback(self) -> None:
+        if not self.get_call_back_attribute():
+            return
+
+        call_back = self.get_call_back_attribute()
+
+        assert isinstance(call_back, str), (
+            "oauth_cb must be a string representing python function reference "
+            "in the format <python-module>:<function-name>."
+        )
+
+        call_back_fn = import_path(call_back)
+        self._validate_call_back_fn_signature(call_back_fn)
+
+        # Set the callback
+        self._config[CallableConsumerConfig.CALLBACK_ATTRIBUTE] = call_back_fn
+
+    def _validate_call_back_fn_signature(self, call_back_fn: Any) -> None:
+        sig = inspect.signature(call_back_fn)
+
+        num_positional_args = len(
+            [
+                param
+                for param in sig.parameters.values()
+                if param.kind
+                in (
+                    inspect.Parameter.POSITIONAL_ONLY,
+                    inspect.Parameter.POSITIONAL_OR_KEYWORD,
+                )
+                and param.default == inspect.Parameter.empty
+            ]
+        )
+
+        has_variadic_args = any(
+            param.kind == inspect.Parameter.VAR_POSITIONAL
+            for param in sig.parameters.values()
+        )
+
+        assert num_positional_args == 1 or (
+            has_variadic_args and num_positional_args <= 1
+        ), "oauth_cb function must accept single positional argument."
diff --git a/metadata-ingestion/src/datahub/configuration/source_common.py b/metadata-ingestion/src/datahub/configuration/source_common.py
index ad12447532335..8e41e9fb91787 100644
--- a/metadata-ingestion/src/datahub/configuration/source_common.py
+++ b/metadata-ingestion/src/datahub/configuration/source_common.py
@@ -1,14 +1,10 @@
-from typing import Dict, Optional, Set
+from typing import Dict, Optional
 
 from pydantic import validator
 from pydantic.fields import Field
 
 from datahub.configuration.common import ConfigModel
-from datahub.emitter.enum_helpers import get_enum_options
-from datahub.metadata.schema_classes import FabricTypeClass
-
-DEFAULT_ENV = FabricTypeClass.PROD
-ALL_ENV_TYPES: Set[str] = set(get_enum_options(FabricTypeClass))
+from datahub.emitter.mce_builder import ALL_ENV_TYPES, DEFAULT_ENV
 
 
 class PlatformInstanceConfigMixin(ConfigModel):
@@ -67,3 +63,16 @@ class DatasetLineageProviderConfigBase(EnvConfigMixin):
         default=None,
         description="A holder for platform -> platform_instance mappings to generate correct dataset urns",
     )
+
+
+class PlatformDetail(ConfigModel):
+    platform_instance: Optional[str] = Field(
+        default=None,
+        description="DataHub platform instance name. To generate correct urn for upstream dataset, this should match "
+        "with platform instance name used in ingestion "
+        "recipe of other datahub sources.",
+    )
+    env: str = Field(
+        default=DEFAULT_ENV,
+        description="The environment that all assets produced by DataHub platform ingestion source belong to",
+    )
diff --git a/metadata-ingestion/src/datahub/configuration/yaml.py b/metadata-ingestion/src/datahub/configuration/yaml.py
index 1f1172836f744..c069845e1de11 100644
--- a/metadata-ingestion/src/datahub/configuration/yaml.py
+++ b/metadata-ingestion/src/datahub/configuration/yaml.py
@@ -2,7 +2,7 @@
 
 import yaml
 
-from datahub.configuration import ConfigurationMechanism
+from datahub.configuration.common import ConfigurationMechanism
 
 
 class YamlConfigurationMechanism(ConfigurationMechanism):
diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py
index d3a930d988171..69946c575908b 100644
--- a/metadata-ingestion/src/datahub/emitter/mce_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py
@@ -13,6 +13,7 @@
     Any,
     List,
     Optional,
+    Set,
     Tuple,
     Type,
     TypeVar,
@@ -24,7 +25,6 @@
 import typing_inspect
 from avrogen.dict_wrapper import DictWrapper
 
-from datahub.configuration.source_common import DEFAULT_ENV
 from datahub.emitter.enum_helpers import get_enum_options
 from datahub.metadata.schema_classes import (
     AssertionKeyClass,
@@ -35,6 +35,7 @@
     DatasetKeyClass,
     DatasetLineageTypeClass,
     DatasetSnapshotClass,
+    FabricTypeClass,
     GlobalTagsClass,
     GlossaryTermAssociationClass,
     GlossaryTermsClass as GlossaryTerms,
@@ -56,6 +57,9 @@
 logger = logging.getLogger(__name__)
 Aspect = TypeVar("Aspect", bound=AspectAbstract)
 
+DEFAULT_ENV = FabricTypeClass.PROD
+ALL_ENV_TYPES: Set[str] = set(get_enum_options(FabricTypeClass))
+
 DEFAULT_FLOW_CLUSTER = "prod"
 UNKNOWN_USER = "urn:li:corpuser:unknown"
 DATASET_URN_TO_LOWER: bool = (
@@ -163,11 +167,15 @@ def dataset_key_to_urn(key: DatasetKeyClass) -> str:
 
 
 def make_container_urn(guid: Union[str, "DatahubKey"]) -> str:
-    from datahub.emitter.mcp_builder import DatahubKey
+    if isinstance(guid, str) and guid.startswith("urn:li:container"):
+        return guid
+    else:
+        from datahub.emitter.mcp_builder import DatahubKey
+
+        if isinstance(guid, DatahubKey):
+            guid = guid.guid()
 
-    if isinstance(guid, DatahubKey):
-        guid = guid.guid()
-    return f"urn:li:container:{guid}"
+        return f"urn:li:container:{guid}"
 
 
 def container_urn_to_key(guid: str) -> Optional[ContainerKeyClass]:
diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
index b7fb1fd56891c..293157f8a1ed0 100644
--- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
@@ -213,6 +213,17 @@ def gen_containers(
     )
 
     container_urn = container_key.as_urn()
+
+    if parent_container_key:  # Yield Container aspect first for auto_browse_path_v2
+        parent_container_urn = make_container_urn(guid=parent_container_key.guid())
+
+        # Set database container
+        parent_container_mcp = MetadataChangeProposalWrapper(
+            entityUrn=f"{container_urn}",
+            aspect=ContainerClass(container=parent_container_urn),
+        )
+        yield parent_container_mcp.as_workunit()
+
     yield MetadataChangeProposalWrapper(
         entityUrn=f"{container_urn}",
         aspect=ContainerProperties(
@@ -276,18 +287,6 @@ def gen_containers(
             tags=sorted(tags),
         )
 
-    if parent_container_key:
-        parent_container_urn = make_container_urn(
-            guid=parent_container_key.guid(),
-        )
-
-        # Set database container
-        parent_container_mcp = MetadataChangeProposalWrapper(
-            entityUrn=f"{container_urn}",
-            aspect=ContainerClass(container=parent_container_urn),
-        )
-        yield parent_container_mcp.as_workunit()
-
 
 def add_dataset_to_container(
     container_key: KeyType, dataset_urn: str
diff --git a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py
index 3790399539437..779b42e1e1ee9 100644
--- a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py
@@ -1,4 +1,5 @@
 import json
+import time
 from collections import defaultdict
 from dataclasses import dataclass
 from typing import Any, Dict, Iterable, List, Optional, Sequence, Union
@@ -6,12 +7,15 @@
 from datahub.emitter.aspect import JSON_PATCH_CONTENT_TYPE
 from datahub.emitter.serialization_helper import pre_json_transform
 from datahub.metadata.schema_classes import (
+    AuditStampClass,
     ChangeTypeClass,
+    EdgeClass,
     GenericAspectClass,
     KafkaAuditHeaderClass,
     MetadataChangeProposalClass,
     SystemMetadataClass,
 )
+from datahub.metadata.urns import Urn
 from datahub.utilities.urns.urn import guess_entity_type
 
 
@@ -89,3 +93,42 @@ def build(self) -> Iterable[MetadataChangeProposalClass]:
             )
             for aspect_name, patches in self.patches.items()
         ]
+
+    @classmethod
+    def _mint_auditstamp(cls, message: Optional[str] = None) -> AuditStampClass:
+        """
+        Creates an AuditStampClass instance with the current timestamp and other default values.
+
+        Args:
+            message: The message associated with the audit stamp (optional).
+
+        Returns:
+            An instance of AuditStampClass.
+        """
+        return AuditStampClass(
+            time=int(time.time() * 1000.0),
+            actor="urn:li:corpuser:datahub",
+            message=message,
+        )
+
+    @classmethod
+    def _ensure_urn_type(
+        cls, entity_type: str, edges: List[EdgeClass], context: str
+    ) -> None:
+        """
+        Ensures that the destination URNs in the given edges have the specified entity type.
+
+        Args:
+            entity_type: The entity type to check against.
+            edges: A list of Edge objects.
+            context: The context or description of the operation.
+
+        Raises:
+            ValueError: If any of the destination URNs is not of the specified entity type.
+        """
+        for e in edges:
+            urn = Urn.from_string(e.destinationUrn)
+            if not urn.entity_type == entity_type:
+                raise ValueError(
+                    f"{context}: {e.destinationUrn} is not of type {entity_type}"
+                )
diff --git a/metadata-ingestion/src/datahub/emitter/rest_emitter.py b/metadata-ingestion/src/datahub/emitter/rest_emitter.py
index 948060c3c4f44..675717b5ec482 100644
--- a/metadata-ingestion/src/datahub/emitter/rest_emitter.py
+++ b/metadata-ingestion/src/datahub/emitter/rest_emitter.py
@@ -10,6 +10,7 @@
 from requests.adapters import HTTPAdapter, Retry
 from requests.exceptions import HTTPError, RequestException
 
+from datahub import nice_version_name
 from datahub.cli import config_utils
 from datahub.cli.cli_utils import ensure_has_system_metadata, fixup_gms_url
 from datahub.configuration.common import ConfigurationError, OperationalError
@@ -45,8 +46,18 @@
     os.getenv("DATAHUB_REST_EMITTER_DEFAULT_RETRY_MAX_TIMES", "4")
 )
 
-# The limit is 16mb. We will use a max of 15mb to have some space for overhead.
-_MAX_BATCH_INGEST_PAYLOAD_SIZE = 15 * 1024 * 1024
+# The limit is 16mb. We will use a max of 15mb to have some space
+# for overhead like request headers.
+# This applies to pretty much all calls to GMS.
+INGEST_MAX_PAYLOAD_BYTES = 15 * 1024 * 1024
+
+# This limit is somewhat arbitrary. All GMS endpoints will timeout
+# and return a 500 if processing takes too long. To avoid sending
+# too much to the backend and hitting a timeout, we try to limit
+# the number of MCPs we send in a batch.
+BATCH_INGEST_MAX_PAYLOAD_LENGTH = int(
+    os.getenv("DATAHUB_REST_EMITTER_BATCH_MAX_PAYLOAD_LENGTH", 200)
+)
 
 
 class DataHubRestEmitter(Closeable, Emitter):
@@ -91,6 +102,7 @@ def __init__(
         self._session.headers.update(
             {
                 "X-RestLi-Protocol-Version": "2.0.0",
+                "X-DataHub-Py-Cli-Version": nice_version_name(),
                 "Content-Type": "application/json",
             }
         )
@@ -279,6 +291,7 @@ def emit_mcps(
         mcps: List[Union[MetadataChangeProposal, MetadataChangeProposalWrapper]],
         async_flag: Optional[bool] = None,
     ) -> int:
+        logger.debug("Attempting to emit batch mcps")
         url = f"{self._gms_server}/aspects?action=ingestProposalBatch"
         for mcp in mcps:
             ensure_has_system_metadata(mcp)
@@ -288,15 +301,25 @@ def emit_mcps(
         # As a safety mechanism, we need to make sure we don't exceed the max payload size for GMS.
         # If we will exceed the limit, we need to break it up into chunks.
         mcp_obj_chunks: List[List[str]] = []
-        current_chunk_size = _MAX_BATCH_INGEST_PAYLOAD_SIZE
+        current_chunk_size = INGEST_MAX_PAYLOAD_BYTES
         for mcp_obj in mcp_objs:
             mcp_obj_size = len(json.dumps(mcp_obj))
+            logger.debug(
+                f"Iterating through object with size {mcp_obj_size} (type: {mcp_obj.get('aspectName')}"
+            )
 
-            if mcp_obj_size + current_chunk_size > _MAX_BATCH_INGEST_PAYLOAD_SIZE:
+            if (
+                mcp_obj_size + current_chunk_size > INGEST_MAX_PAYLOAD_BYTES
+                or len(mcp_obj_chunks[-1]) >= BATCH_INGEST_MAX_PAYLOAD_LENGTH
+            ):
+                logger.debug("Decided to create new chunk")
                 mcp_obj_chunks.append([])
                 current_chunk_size = 0
             mcp_obj_chunks[-1].append(mcp_obj)
             current_chunk_size += mcp_obj_size
+        logger.debug(
+            f"Decided to send {len(mcps)} mcps in {len(mcp_obj_chunks)} chunks"
+        )
 
         for mcp_obj_chunk in mcp_obj_chunks:
             # TODO: We're calling json.dumps on each MCP object twice, once to estimate
@@ -323,8 +346,15 @@ def emit_usage(self, usageStats: UsageAggregation) -> None:
 
     def _emit_generic(self, url: str, payload: str) -> None:
         curl_command = make_curl_command(self._session, "POST", url, payload)
+        payload_size = len(payload)
+        if payload_size > INGEST_MAX_PAYLOAD_BYTES:
+            # since we know total payload size here, we could simply avoid sending such payload at all and report a warning, with current approach we are going to cause whole ingestion to fail
+            logger.warning(
+                f"Apparent payload size exceeded {INGEST_MAX_PAYLOAD_BYTES}, might fail with an exception due to the size"
+            )
         logger.debug(
-            "Attempting to emit to DataHub GMS; using curl equivalent to:\n%s",
+            "Attempting to emit aspect (size: %s) to DataHub GMS; using curl equivalent to:\n%s",
+            payload_size,
             curl_command,
         )
         try:
diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py
index d088380d5d38c..85968f050a371 100644
--- a/metadata-ingestion/src/datahub/entrypoints.py
+++ b/metadata-ingestion/src/datahub/entrypoints.py
@@ -13,13 +13,10 @@
     generate_access_token,
     make_shim_command,
 )
-from datahub.cli.config_utils import (
-    DATAHUB_CONFIG_PATH,
-    get_boolean_env_variable,
-    write_gms_config,
-)
+from datahub.cli.config_utils import DATAHUB_CONFIG_PATH, write_gms_config
 from datahub.cli.delete_cli import delete
 from datahub.cli.docker_cli import docker
+from datahub.cli.env_utils import get_boolean_env_variable
 from datahub.cli.exists_cli import exists
 from datahub.cli.get_cli import get
 from datahub.cli.ingest_cli import ingest
diff --git a/metadata-ingestion/src/datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py b/metadata-ingestion/src/datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py
new file mode 100644
index 0000000000000..559f0b77f59df
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py
@@ -0,0 +1,96 @@
+import json
+import logging
+from typing import Iterable, List
+
+from datahub.emitter.rest_emitter import INGEST_MAX_PAYLOAD_BYTES
+from datahub.emitter.serialization_helper import pre_json_transform
+from datahub.ingestion.api.source import SourceReport
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.metadata.schema_classes import (
+    DatasetProfileClass,
+    SchemaFieldClass,
+    SchemaMetadataClass,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class EnsureAspectSizeProcessor:
+    def __init__(
+        self, report: SourceReport, payload_constraint: int = INGEST_MAX_PAYLOAD_BYTES
+    ):
+        self.report = report
+        self.payload_constraint = payload_constraint
+
+    def ensure_dataset_profile_size(
+        self, dataset_urn: str, profile: DatasetProfileClass
+    ) -> None:
+        """
+        This is quite arbitrary approach to ensuring dataset profile aspect does not exceed allowed size, might be adjusted
+        in the future
+        """
+        sample_fields_size = 0
+        if profile.fieldProfiles:
+            logger.debug(f"Length of field profiles: {len(profile.fieldProfiles)}")
+            for field in profile.fieldProfiles:
+                if field.sampleValues:
+                    values_len = 0
+                    for value in field.sampleValues:
+                        if value:
+                            values_len += len(value)
+                    logger.debug(
+                        f"Field {field.fieldPath} has {len(field.sampleValues)} sample values, taking total bytes {values_len}"
+                    )
+                    if sample_fields_size + values_len > self.payload_constraint:
+                        field.sampleValues = []
+                        self.report.warning(
+                            title="Dataset profile truncated due to size constraint",
+                            message="Dataset profile contained too much data and would have caused ingestion to fail",
+                            context=f"Sample values for field {field.fieldPath} were removed from dataset profile for {dataset_urn} due to aspect size constraints",
+                        )
+                    else:
+                        sample_fields_size += values_len
+                else:
+                    logger.debug(f"Field {field.fieldPath} has no sample values")
+
+    def ensure_schema_metadata_size(
+        self, dataset_urn: str, schema: SchemaMetadataClass
+    ) -> None:
+        """
+        This is quite arbitrary approach to ensuring schema metadata aspect does not exceed allowed size, might be adjusted
+        in the future
+        """
+        total_fields_size = 0
+        logger.debug(f"Amount of schema fields: {len(schema.fields)}")
+        accepted_fields: List[SchemaFieldClass] = []
+        for field in schema.fields:
+            field_size = len(json.dumps(pre_json_transform(field.to_obj())))
+            logger.debug(f"Field {field.fieldPath} takes total {field_size}")
+            if total_fields_size + field_size < self.payload_constraint:
+                accepted_fields.append(field)
+                total_fields_size += field_size
+            else:
+                self.report.warning(
+                    title="Schema truncated due to size constraint",
+                    message="Dataset schema contained too much data and would have caused ingestion to fail",
+                    context=f"Field {field.fieldPath} was removed from schema for {dataset_urn} due to aspect size constraints",
+                )
+
+        schema.fields = accepted_fields
+
+    def ensure_aspect_size(
+        self,
+        stream: Iterable[MetadataWorkUnit],
+    ) -> Iterable[MetadataWorkUnit]:
+        """
+        We have hard limitation of aspect size being 16 MB. Some aspects can exceed that value causing an exception
+        on GMS side and failure of the entire ingestion. This processor will attempt to trim suspected aspects.
+        """
+        for wu in stream:
+            logger.debug(f"Ensuring size of workunit: {wu.id}")
+
+            if schema := wu.get_aspect_of_type(SchemaMetadataClass):
+                self.ensure_schema_metadata_size(wu.get_urn(), schema)
+            elif profile := wu.get_aspect_of_type(DatasetProfileClass):
+                self.ensure_dataset_profile_size(wu.get_urn(), profile)
+            yield wu
diff --git a/metadata-ingestion/src/datahub/ingestion/api/decorators.py b/metadata-ingestion/src/datahub/ingestion/api/decorators.py
index b390ffb9dd036..d32c0b85ceef4 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/decorators.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/decorators.py
@@ -3,7 +3,10 @@
 from typing import Callable, Dict, Optional, Type
 
 from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.api.source import Source, SourceCapability
+from datahub.ingestion.api.source import (  # noqa: I250
+    Source,
+    SourceCapability as SourceCapability,
+)
 
 
 def config_class(config_cls: Type) -> Callable[[Type], Type]:
diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_properties_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_properties_helper.py
new file mode 100644
index 0000000000000..151b0c72a6c2d
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_properties_helper.py
@@ -0,0 +1,69 @@
+import logging
+from typing import Iterable, Optional
+
+from pydantic.fields import Field
+
+from datahub.configuration.common import ConfigModel
+from datahub.emitter.mce_builder import set_aspect
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.source_helpers import create_dataset_props_patch_builder
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.metadata.schema_classes import (
+    DatasetPropertiesClass,
+    MetadataChangeEventClass,
+    SystemMetadataClass,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def convert_dataset_properties_to_patch(
+    urn: str,
+    aspect: DatasetPropertiesClass,
+    system_metadata: Optional[SystemMetadataClass],
+) -> MetadataWorkUnit:
+    patch_builder = create_dataset_props_patch_builder(urn, aspect, system_metadata)
+    mcp = next(iter(patch_builder.build()))
+    return MetadataWorkUnit(id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp)
+
+
+def auto_incremental_properties(
+    incremental_properties: bool,
+    stream: Iterable[MetadataWorkUnit],
+) -> Iterable[MetadataWorkUnit]:
+    if not incremental_properties:
+        yield from stream
+        return  # early exit
+
+    for wu in stream:
+        urn = wu.get_urn()
+
+        if isinstance(wu.metadata, MetadataChangeEventClass):
+            properties_aspect = wu.get_aspect_of_type(DatasetPropertiesClass)
+            set_aspect(wu.metadata, None, DatasetPropertiesClass)
+            if len(wu.metadata.proposedSnapshot.aspects) > 0:
+                yield wu
+
+            if properties_aspect:
+                yield convert_dataset_properties_to_patch(
+                    urn, properties_aspect, wu.metadata.systemMetadata
+                )
+        elif isinstance(wu.metadata, MetadataChangeProposalWrapper) and isinstance(
+            wu.metadata.aspect, DatasetPropertiesClass
+        ):
+            properties_aspect = wu.metadata.aspect
+            if properties_aspect:
+                yield convert_dataset_properties_to_patch(
+                    urn, properties_aspect, wu.metadata.systemMetadata
+                )
+        else:
+            yield wu
+
+
+# TODO: Use this in SQLCommonConfig. Currently only used in snowflake
+class IncrementalPropertiesConfigMixin(ConfigModel):
+    incremental_properties: bool = Field(
+        default=False,
+        description="When enabled, emits dataset properties as incremental to existing dataset properties "
+        "in DataHub. When disabled, re-states dataset properties on each run.",
+    )
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py
index 85ae17ddf6529..c3638635b19aa 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source.py
@@ -37,6 +37,7 @@
 from datahub.ingestion.api.source_helpers import (
     auto_browse_path_v2,
     auto_fix_duplicate_schema_field_paths,
+    auto_fix_empty_field_paths,
     auto_lowercase_urns,
     auto_materialize_referenced_tags_terms,
     auto_status_aspect,
@@ -183,6 +184,7 @@ def infos(self) -> LossyList[StructuredLogEntry]:
 
 @dataclass
 class SourceReport(Report):
+    event_not_produced_warn: bool = True
     events_produced: int = 0
     events_produced_per_sec: int = 0
 
@@ -444,6 +446,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
             partial(
                 auto_fix_duplicate_schema_field_paths, platform=self._infer_platform()
             ),
+            partial(auto_fix_empty_field_paths, platform=self._infer_platform()),
             browse_path_processor,
             partial(auto_workunit_reporter, self.get_report()),
             auto_patch_last_modified,
@@ -490,11 +493,15 @@ def close(self) -> None:
 
     def _infer_platform(self) -> Optional[str]:
         config = self.get_config()
-        return (
+        platform = (
             getattr(config, "platform_name", None)
             or getattr(self, "platform", None)
             or getattr(config, "platform", None)
         )
+        if platform is None and hasattr(self, "get_platform_id"):
+            platform = type(self).get_platform_id()
+
+        return platform
 
     def _get_browse_path_processor(self, dry_run: bool) -> MetadataWorkUnitProcessor:
         config = self.get_config()
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
index 372aef707f232..7791ea2797be3 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
@@ -32,6 +32,7 @@
     SchemaFieldClass,
     SchemaMetadataClass,
     StatusClass,
+    SystemMetadataClass,
     TimeWindowSizeClass,
 )
 from datahub.metadata.urns import DatasetUrn, GlossaryTermUrn, TagUrn, Urn
@@ -65,9 +66,10 @@ def auto_workunit(
 def create_dataset_props_patch_builder(
     dataset_urn: str,
     dataset_properties: DatasetPropertiesClass,
+    system_metadata: Optional[SystemMetadataClass] = None,
 ) -> DatasetPatchBuilder:
     """Creates a patch builder with a table's or view's attributes and dataset properties"""
-    patch_builder = DatasetPatchBuilder(dataset_urn)
+    patch_builder = DatasetPatchBuilder(dataset_urn, system_metadata)
     patch_builder.set_display_name(dataset_properties.name)
     patch_builder.set_description(dataset_properties.description)
     patch_builder.set_created(dataset_properties.created)
@@ -148,6 +150,12 @@ def auto_workunit_reporter(report: "SourceReport", stream: Iterable[T]) -> Itera
         report.report_workunit(wu)
         yield wu
 
+    if report.event_not_produced_warn and report.events_produced == 0:
+        report.warning(
+            title="No metadata was produced by the source",
+            message="Please check the source configuration, filters, and permissions.",
+        )
+
 
 def auto_materialize_referenced_tags_terms(
     stream: Iterable[MetadataWorkUnit],
@@ -394,6 +402,50 @@ def auto_fix_duplicate_schema_field_paths(
         )
 
 
+def auto_fix_empty_field_paths(
+    stream: Iterable[MetadataWorkUnit],
+    *,
+    platform: Optional[str] = None,
+) -> Iterable[MetadataWorkUnit]:
+    """Count schema metadata aspects with empty field paths and emit telemetry."""
+
+    total_schema_aspects = 0
+    schemas_with_empty_fields = 0
+    empty_field_paths = 0
+
+    for wu in stream:
+        schema_metadata = wu.get_aspect_of_type(SchemaMetadataClass)
+        if schema_metadata:
+            total_schema_aspects += 1
+
+            updated_fields: List[SchemaFieldClass] = []
+            for field in schema_metadata.fields:
+                if field.fieldPath:
+                    updated_fields.append(field)
+                else:
+                    empty_field_paths += 1
+
+            if empty_field_paths > 0:
+                logger.info(
+                    f"Fixing empty field paths in schema aspect for {wu.get_urn()} by dropping empty fields"
+                )
+                schema_metadata.fields = updated_fields
+                schemas_with_empty_fields += 1
+
+        yield wu
+
+    if schemas_with_empty_fields > 0:
+        properties = {
+            "platform": platform,
+            "total_schema_aspects": total_schema_aspects,
+            "schemas_with_empty_fields": schemas_with_empty_fields,
+            "empty_field_paths": empty_field_paths,
+        }
+        telemetry.telemetry_instance.ping(
+            "ingestion_empty_schema_field_paths", properties
+        )
+
+
 def auto_empty_dataset_usage_statistics(
     stream: Iterable[MetadataWorkUnit],
     *,
diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py
index bcf077154343c..88d1fcc52e219 100644
--- a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py
+++ b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py
@@ -23,7 +23,7 @@
     RecordTypeClass,
     SchemaFieldClass as SchemaField,
     SchemaFieldDataTypeClass,
-    SchemaMetadataClass as SchemaMetadata,
+    SchemaMetadataClass,
     StringTypeClass,
     UnionTypeClass,
 )
@@ -665,13 +665,13 @@ def get_schema_metadata(
     name: str,
     json_schema: Dict[Any, Any],
     raw_schema_string: Optional[str] = None,
-) -> SchemaMetadata:
+) -> SchemaMetadataClass:
     json_schema_as_string = raw_schema_string or json.dumps(json_schema)
     md5_hash: str = md5(json_schema_as_string.encode()).hexdigest()
 
     schema_fields = list(JsonSchemaTranslator.get_fields_from_schema(json_schema))
 
-    schema_metadata = SchemaMetadata(
+    schema_metadata = SchemaMetadataClass(
         schemaName=name,
         platform=f"urn:li:dataPlatform:{platform}",
         version=0,
diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py
index f62bb184252d9..e947aff384871 100644
--- a/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py
+++ b/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py
@@ -32,7 +32,7 @@
     OneofDescriptor,
 )
 
-from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     BooleanTypeClass,
     BytesTypeClass,
@@ -41,8 +41,8 @@
     MapTypeClass,
     NumberTypeClass,
     RecordTypeClass,
-    SchemaField,
-    SchemaFieldDataType,
+    SchemaFieldClass as SchemaField,
+    SchemaFieldDataTypeClass as SchemaFieldDataType,
     StringTypeClass,
     UnionTypeClass,
 )
diff --git a/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py b/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py
index a135b7b6ce837..9c34c4f83b0a9 100644
--- a/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py
+++ b/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py
@@ -32,7 +32,6 @@ def __str__(self):
 
 
 class S3ListIterator(Iterator):
-
     MAX_KEYS = 1000
 
     def __init__(
diff --git a/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py b/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py
index 1d381acbf3dbe..98c43079a3bc1 100644
--- a/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py
+++ b/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py
@@ -33,7 +33,6 @@
 
 @dataclass
 class ClassificationReportMixin:
-
     num_tables_fetch_sample_values_failed: int = 0
 
     num_tables_classification_attempted: int = 0
@@ -112,7 +111,6 @@ def classify_schema_fields(
         schema_metadata: SchemaMetadata,
         sample_data: Union[Dict[str, list], Callable[[], Dict[str, list]]],
     ) -> None:
-
         if not isinstance(sample_data, Dict):
             try:
                 # TODO: In future, sample_data fetcher can be lazily called if classification
diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py
index b9b0ed556e66c..ca9a41172e5b6 100644
--- a/metadata-ingestion/src/datahub/ingestion/graph/client.py
+++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py
@@ -33,7 +33,9 @@
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.rest_emitter import DatahubRestEmitter
 from datahub.emitter.serialization_helper import post_json_transform
-from datahub.ingestion.graph.config import DatahubClientConfig
+from datahub.ingestion.graph.config import (  # noqa: I250; TODO: Remove this alias
+    DatahubClientConfig as DatahubClientConfig,
+)
 from datahub.ingestion.graph.connections import (
     connections_gql,
     get_id_from_connection_urn,
@@ -65,6 +67,7 @@
     SystemMetadataClass,
     TelemetryClientIdClass,
 )
+from datahub.telemetry.telemetry import telemetry_instance
 from datahub.utilities.perf_timer import PerfTimer
 from datahub.utilities.str_enum import StrEnum
 from datahub.utilities.urns.urn import Urn, guess_entity_type
@@ -185,9 +188,12 @@ def from_emitter(cls, emitter: DatahubRestEmitter) -> "DataHubGraph":
                 retry_max_times=emitter._retry_max_times,
                 extra_headers=emitter._session.headers,
                 disable_ssl_verification=emitter._session.verify is False,
-                # TODO: Support these headers.
-                # ca_certificate_path=emitter._ca_certificate_path,
-                # client_certificate_path=emitter._client_certificate_path,
+                ca_certificate_path=(
+                    emitter._session.verify
+                    if isinstance(emitter._session.verify, str)
+                    else None
+                ),
+                client_certificate_path=emitter._session.cert,
             )
         )
 
@@ -351,6 +357,7 @@ def get_tags(self, entity_urn: str) -> Optional[GlobalTagsClass]:
     def get_glossary_terms(self, entity_urn: str) -> Optional[GlossaryTermsClass]:
         return self.get_aspect(entity_urn=entity_urn, aspect_type=GlossaryTermsClass)
 
+    @functools.lru_cache(maxsize=1)
     def get_domain(self, entity_urn: str) -> Optional[DomainsClass]:
         return self.get_aspect(entity_urn=entity_urn, aspect_type=DomainsClass)
 
@@ -559,8 +566,10 @@ def get_entity_as_mcps(
             post_json_obj = post_json_transform(aspect_json)
             aspect_value = aspect_type.from_obj(post_json_obj["value"])
 
-            system_metadata_raw = post_json_obj["systemMetadata"]
-            system_metadata = SystemMetadataClass.from_obj(system_metadata_raw)
+            system_metadata_raw = post_json_obj.get("systemMetadata")
+            system_metadata = None
+            if system_metadata_raw:
+                system_metadata = SystemMetadataClass.from_obj(system_metadata_raw)
 
             mcpw = MetadataChangeProposalWrapper(
                 entityUrn=entity_urn,
@@ -590,7 +599,7 @@ def get_entity_semityped(
             not be present in the dictionary. The entity's key aspect will always be present.
         """
 
-        mcps = self.get_entity_as_mcps(entity_urn, aspects)
+        mcps = self.get_entity_as_mcps(entity_urn, aspects=aspects)
 
         result: AspectBag = {}
         for mcp in mcps:
@@ -1238,14 +1247,29 @@ def soft_delete_entity(
         Args:
             urn: The urn of the entity to soft-delete.
         """
+        self.set_soft_delete_status(
+            urn=urn, run_id=run_id, deletion_timestamp=deletion_timestamp, delete=True
+        )
+
+    def set_soft_delete_status(
+        self,
+        urn: str,
+        delete: bool,
+        run_id: str = _GRAPH_DUMMY_RUN_ID,
+        deletion_timestamp: Optional[int] = None,
+    ) -> None:
+        """Change status of soft-delete an entity by urn.
 
+        Args:
+            urn: The urn of the entity to soft-delete.
+        """
         assert urn
 
         deletion_timestamp = deletion_timestamp or int(time.time() * 1000)
         self.emit(
             MetadataChangeProposalWrapper(
                 entityUrn=urn,
-                aspect=StatusClass(removed=True),
+                aspect=StatusClass(removed=delete),
                 systemMetadata=SystemMetadataClass(
                     runId=run_id, lastObserved=deletion_timestamp
                 ),
@@ -1799,4 +1823,5 @@ def get_default_graph() -> DataHubGraph:
     graph_config = config_utils.load_client_config()
     graph = DataHubGraph(graph_config)
     graph.test_connection()
+    telemetry_instance.set_context(server=graph)
     return graph
diff --git a/metadata-ingestion/src/datahub/ingestion/graph/config.py b/metadata-ingestion/src/datahub/ingestion/graph/config.py
index cf0ec45b71458..5f269e14e1a4a 100644
--- a/metadata-ingestion/src/datahub/ingestion/graph/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/graph/config.py
@@ -8,7 +8,7 @@ class DatahubClientConfig(ConfigModel):
 
     # TODO: Having a default for the server doesn't make a ton of sense. This should be handled
     # by callers / the CLI, but the actual client should not have any magic.
-    server: str = "http://localhost:8080"
+    server: str
     token: Optional[str] = None
     timeout_sec: Optional[int] = None
     retry_status_codes: Optional[List[int]] = None
diff --git a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py
index 33bfb63feb3fd..28def68ccf3f5 100644
--- a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py
+++ b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py
@@ -11,9 +11,8 @@
     redact_raw_config,
 )
 from datahub.emitter.aspect import JSON_CONTENT_TYPE
-from datahub.emitter.mce_builder import datahub_guid
+from datahub.emitter.mce_builder import datahub_guid, make_data_platform_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.emitter.mcp_builder import make_data_platform_urn
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
 from datahub.ingestion.api.pipeline_run_listener import PipelineRunListener
 from datahub.ingestion.api.sink import NoopWriteCallback, Sink
@@ -149,10 +148,10 @@ def __init__(self, sink: Sink, report_recipe: bool, ctx: PipelineContext) -> Non
 
     def _get_recipe_to_report(self, ctx: PipelineContext) -> str:
         assert ctx.pipeline_config
-        if not self.report_recipe or not ctx.pipeline_config._raw_dict:
+        if not self.report_recipe or not ctx.pipeline_config.get_raw_dict():
             return ""
         else:
-            return json.dumps(redact_raw_config(ctx.pipeline_config._raw_dict))
+            return json.dumps(redact_raw_config(ctx.pipeline_config.get_raw_dict()))
 
     def _emit_aspect(self, entity_urn: Urn, aspect_value: _Aspect) -> None:
         self.sink.write_record_async(
diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
index a16a3df57d1bc..ee1c1608cd48c 100644
--- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
+++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
@@ -44,7 +44,8 @@
 )
 from datahub.ingestion.transformer.transform_registry import transform_registry
 from datahub.metadata.schema_classes import MetadataChangeProposalClass
-from datahub.telemetry import stats, telemetry
+from datahub.telemetry import stats
+from datahub.telemetry.telemetry import telemetry_instance
 from datahub.utilities._custom_package_loader import model_version_name
 from datahub.utilities.global_warning_util import (
     clear_global_warnings,
@@ -220,7 +221,7 @@ def __init__(
         dry_run: bool = False,
         preview_mode: bool = False,
         preview_workunits: int = 10,
-        report_to: Optional[str] = None,
+        report_to: Optional[str] = "datahub",
         no_progress: bool = False,
     ):
         self.config = config
@@ -273,8 +274,9 @@ def __init__(
         if self.graph is None and isinstance(self.sink, DatahubRestSink):
             with _add_init_error_context("setup default datahub client"):
                 self.graph = self.sink.emitter.to_graph()
+                self.graph.test_connection()
         self.ctx.graph = self.graph
-        telemetry.telemetry_instance.update_capture_exception_context(server=self.graph)
+        telemetry_instance.set_context(server=self.graph)
 
         with set_graph_context(self.graph):
             with _add_init_error_context("configure reporters"):
@@ -428,6 +430,7 @@ def create(
     def _time_to_print(self) -> bool:
         self.num_intermediate_workunits += 1
         current_time = int(time.time())
+        # TODO: Replace with ProgressTimer.
         if current_time - self.last_time_printed > _REPORT_PRINT_INTERVAL_SECONDS:
             # we print
             self.num_intermediate_workunits = 0
@@ -614,7 +617,7 @@ def log_ingestion_stats(self) -> None:
         sink_warnings = len(self.sink.get_report().warnings)
         global_warnings = len(get_global_warnings())
 
-        telemetry.telemetry_instance.ping(
+        telemetry_instance.ping(
             "ingest_stats",
             {
                 "source_type": self.source_type,
@@ -636,7 +639,6 @@ def log_ingestion_stats(self) -> None:
                 ),
                 "has_pipeline_name": bool(self.config.pipeline_name),
             },
-            self.ctx.graph,
         )
 
     def _approx_all_vals(self, d: LossyList[Any]) -> int:
@@ -674,7 +676,7 @@ def pretty_print_summary(
         else:
             click.echo()
             click.secho("Cli report:", bold=True)
-            click.secho(self.cli_report.as_string())
+            click.echo(self.cli_report.as_string())
             click.secho(f"Source ({self.source_type}) report:", bold=True)
             click.echo(self.source.get_report().as_string())
             click.secho(f"Sink ({self.sink_type}) report:", bold=True)
@@ -735,11 +737,14 @@ def pretty_print_summary(
             return 0
 
     def _handle_uncaught_pipeline_exception(self, exc: Exception) -> None:
-        logger.exception("Ingestion pipeline threw an uncaught exception")
+        logger.exception(
+            f"Ingestion pipeline threw an uncaught exception: {exc}", stacklevel=2
+        )
         self.source.get_report().report_failure(
             title="Pipeline Error",
             message="Ingestion pipeline raised an unexpected exception!",
             exc=exc,
+            log=False,
         )
 
     def _get_structured_report(self) -> Dict[str, Any]:
diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
index 98629ba030695..7a4e7ec52a8e9 100644
--- a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
@@ -1,6 +1,7 @@
 import datetime
 import logging
-import uuid
+import random
+import string
 from typing import Any, Dict, List, Optional
 
 from pydantic import Field, validator
@@ -71,6 +72,15 @@ class FlagsConfig(ConfigModel):
     )
 
 
+def _generate_run_id(source_type: Optional[str] = None) -> str:
+    current_time = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S")
+    random_suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
+
+    if source_type is None:
+        source_type = "ingestion"
+    return f"{source_type}-{current_time}-{random_suffix}"
+
+
 class PipelineConfig(ConfigModel):
     source: SourceConfig
     sink: Optional[DynamicTypedConfig] = None
@@ -91,12 +101,11 @@ def run_id_should_be_semantic(
         cls, v: Optional[str], values: Dict[str, Any], **kwargs: Any
     ) -> str:
         if v == DEFAULT_RUN_ID:
+            source_type = None
             if "source" in values and hasattr(values["source"], "type"):
                 source_type = values["source"].type
-                current_time = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S")
-                return f"{source_type}-{current_time}"
 
-            return str(uuid.uuid1())  # default run_id if we cannot infer a source type
+            return _generate_run_id(source_type)
         else:
             assert v is not None
             return v
@@ -108,3 +117,9 @@ def from_dict(
         config = cls.parse_obj(resolved_dict)
         config._raw_dict = raw_dict
         return config
+
+    def get_raw_dict(self) -> Dict:
+        result = self._raw_dict
+        if result is None:
+            result = self.dict()
+        return result
diff --git a/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py b/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py
index 5b4d3fe38ecd9..209efbbb90feb 100644
--- a/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py
+++ b/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py
@@ -18,7 +18,10 @@
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import mcps_from_mce
-from datahub.emitter.rest_emitter import DataHubRestEmitter
+from datahub.emitter.rest_emitter import (
+    BATCH_INGEST_MAX_PAYLOAD_LENGTH,
+    DataHubRestEmitter,
+)
 from datahub.ingestion.api.common import RecordEnvelope, WorkUnit
 from datahub.ingestion.api.sink import (
     NoopWriteCallback,
@@ -65,11 +68,19 @@ class DatahubRestSinkConfig(DatahubClientConfig):
     mode: RestSinkMode = _DEFAULT_REST_SINK_MODE
 
     # These only apply in async modes.
-    max_threads: int = _DEFAULT_REST_SINK_MAX_THREADS
-    max_pending_requests: int = 2000
+    max_threads: pydantic.PositiveInt = _DEFAULT_REST_SINK_MAX_THREADS
+    max_pending_requests: pydantic.PositiveInt = 2000
 
     # Only applies in async batch mode.
-    max_per_batch: int = 100
+    max_per_batch: pydantic.PositiveInt = 100
+
+    @pydantic.validator("max_per_batch", always=True)
+    def validate_max_per_batch(cls, v):
+        if v > BATCH_INGEST_MAX_PAYLOAD_LENGTH:
+            raise ValueError(
+                f"max_per_batch must be less than or equal to {BATCH_INGEST_MAX_PAYLOAD_LENGTH}"
+            )
+        return v
 
 
 @dataclasses.dataclass
diff --git a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
index 66f268799b2f1..ad2bc36cf558b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
@@ -201,6 +201,10 @@ def get_fields(self, table_data: TableData, path_spec: PathSpec) -> List:
                 ).infer_schema(file)
             elif extension == ".json":
                 fields = json.JsonInferrer().infer_schema(file)
+            elif extension == ".jsonl":
+                fields = json.JsonInferrer(
+                    max_rows=self.source_config.max_rows, format="jsonl"
+                ).infer_schema(file)
             elif extension == ".avro":
                 fields = avro.AvroInferrer().infer_schema(file)
             else:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
index ce45a5c9b95dc..b76eb95def1ed 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
@@ -1,7 +1,12 @@
+import logging
+import os
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from enum import Enum
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
 
 import boto3
+import requests
 from boto3.session import Session
 from botocore.config import DEFAULT_TIMEOUT, Config
 from botocore.utils import fix_s3_host
@@ -14,6 +19,8 @@
 )
 from datahub.configuration.source_common import EnvConfigMixin
 
+logger = logging.getLogger(__name__)
+
 if TYPE_CHECKING:
     from mypy_boto3_dynamodb import DynamoDBClient
     from mypy_boto3_glue import GlueClient
@@ -22,6 +29,26 @@
     from mypy_boto3_sts import STSClient
 
 
+class AwsEnvironment(Enum):
+    EC2 = "EC2"
+    ECS = "ECS"
+    EKS = "EKS"
+    LAMBDA = "LAMBDA"
+    APP_RUNNER = "APP_RUNNER"
+    BEANSTALK = "ELASTIC_BEANSTALK"
+    CLOUD_FORMATION = "CLOUD_FORMATION"
+    UNKNOWN = "UNKNOWN"
+
+
+class AwsServicePrincipal(Enum):
+    LAMBDA = "lambda.amazonaws.com"
+    EKS = "eks.amazonaws.com"
+    APP_RUNNER = "apprunner.amazonaws.com"
+    ECS = "ecs.amazonaws.com"
+    ELASTIC_BEANSTALK = "elasticbeanstalk.amazonaws.com"
+    EC2 = "ec2.amazonaws.com"
+
+
 class AwsAssumeRoleConfig(PermissiveConfigModel):
     # Using the PermissiveConfigModel to allow the user to pass additional arguments.
 
@@ -34,6 +61,163 @@ class AwsAssumeRoleConfig(PermissiveConfigModel):
     )
 
 
+def get_instance_metadata_token() -> Optional[str]:
+    """Get IMDSv2 token"""
+    try:
+        response = requests.put(
+            "http://169.254.169.254/latest/api/token",
+            headers={"X-aws-ec2-metadata-token-ttl-seconds": "21600"},
+            timeout=1,
+        )
+        if response.status_code == HTTPStatus.OK:
+            return response.text
+    except requests.exceptions.RequestException:
+        logger.debug("Failed to get IMDSv2 token")
+    return None
+
+
+def is_running_on_ec2() -> bool:
+    """Check if code is running on EC2 using IMDSv2"""
+    token = get_instance_metadata_token()
+    if not token:
+        return False
+
+    try:
+        response = requests.get(
+            "http://169.254.169.254/latest/meta-data/instance-id",
+            headers={"X-aws-ec2-metadata-token": token},
+            timeout=1,
+        )
+        return response.status_code == HTTPStatus.OK
+    except requests.exceptions.RequestException:
+        return False
+
+
+def detect_aws_environment() -> AwsEnvironment:
+    """
+    Detect the AWS environment we're running in.
+    Order matters as some environments may have multiple indicators.
+    """
+    # Check Lambda first as it's most specific
+    if os.getenv("AWS_LAMBDA_FUNCTION_NAME"):
+        if os.getenv("AWS_EXECUTION_ENV", "").startswith("CloudFormation"):
+            return AwsEnvironment.CLOUD_FORMATION
+        return AwsEnvironment.LAMBDA
+
+    # Check EKS (IRSA)
+    if os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE") and os.getenv("AWS_ROLE_ARN"):
+        return AwsEnvironment.EKS
+
+    # Check App Runner
+    if os.getenv("AWS_APP_RUNNER_SERVICE_ID"):
+        return AwsEnvironment.APP_RUNNER
+
+    # Check ECS
+    if os.getenv("ECS_CONTAINER_METADATA_URI_V4") or os.getenv(
+        "ECS_CONTAINER_METADATA_URI"
+    ):
+        return AwsEnvironment.ECS
+
+    # Check Elastic Beanstalk
+    if os.getenv("ELASTIC_BEANSTALK_ENVIRONMENT_NAME"):
+        return AwsEnvironment.BEANSTALK
+
+    if is_running_on_ec2():
+        return AwsEnvironment.EC2
+
+    return AwsEnvironment.UNKNOWN
+
+
+def get_instance_role_arn() -> Optional[str]:
+    """Get role ARN from EC2 instance metadata using IMDSv2"""
+    token = get_instance_metadata_token()
+    if not token:
+        return None
+
+    try:
+        response = requests.get(
+            "http://169.254.169.254/latest/meta-data/iam/security-credentials/",
+            headers={"X-aws-ec2-metadata-token": token},
+            timeout=1,
+        )
+        if response.status_code == 200:
+            role_name = response.text.strip()
+            if role_name:
+                sts = boto3.client("sts")
+                identity = sts.get_caller_identity()
+                return identity.get("Arn")
+    except Exception as e:
+        logger.debug(f"Failed to get instance role ARN: {e}")
+    return None
+
+
+def get_lambda_role_arn() -> Optional[str]:
+    """Get the Lambda function's role ARN"""
+    try:
+        function_name = os.getenv("AWS_LAMBDA_FUNCTION_NAME")
+        if not function_name:
+            return None
+
+        lambda_client = boto3.client("lambda")
+        function_config = lambda_client.get_function_configuration(
+            FunctionName=function_name
+        )
+        return function_config.get("Role")
+    except Exception as e:
+        logger.debug(f"Failed to get Lambda role ARN: {e}")
+        return None
+
+
+def get_current_identity() -> Tuple[Optional[str], Optional[str]]:
+    """
+    Get the current role ARN and source type based on the runtime environment.
+    Returns (role_arn, credential_source)
+    """
+    env = detect_aws_environment()
+
+    if env == AwsEnvironment.LAMBDA:
+        role_arn = get_lambda_role_arn()
+        return role_arn, AwsServicePrincipal.LAMBDA.value
+
+    elif env == AwsEnvironment.EKS:
+        role_arn = os.getenv("AWS_ROLE_ARN")
+        return role_arn, AwsServicePrincipal.EKS.value
+
+    elif env == AwsEnvironment.APP_RUNNER:
+        try:
+            sts = boto3.client("sts")
+            identity = sts.get_caller_identity()
+            return identity.get("Arn"), AwsServicePrincipal.APP_RUNNER.value
+        except Exception as e:
+            logger.debug(f"Failed to get App Runner role: {e}")
+
+    elif env == AwsEnvironment.ECS:
+        try:
+            metadata_uri = os.getenv("ECS_CONTAINER_METADATA_URI_V4") or os.getenv(
+                "ECS_CONTAINER_METADATA_URI"
+            )
+            if metadata_uri:
+                response = requests.get(f"{metadata_uri}/task", timeout=1)
+                if response.status_code == HTTPStatus.OK:
+                    task_metadata = response.json()
+                    if "TaskARN" in task_metadata:
+                        return (
+                            task_metadata.get("TaskARN"),
+                            AwsServicePrincipal.ECS.value,
+                        )
+        except Exception as e:
+            logger.debug(f"Failed to get ECS task role: {e}")
+
+    elif env == AwsEnvironment.BEANSTALK:
+        # Beanstalk uses EC2 instance metadata
+        return get_instance_role_arn(), AwsServicePrincipal.ELASTIC_BEANSTALK.value
+
+    elif env == AwsEnvironment.EC2:
+        return get_instance_role_arn(), AwsServicePrincipal.EC2.value
+
+    return None, None
+
+
 def assume_role(
     role: AwsAssumeRoleConfig,
     aws_region: Optional[str],
@@ -95,7 +279,7 @@ class AwsConnectionConfig(ConfigModel):
     )
     aws_profile: Optional[str] = Field(
         default=None,
-        description="Named AWS profile to use. Only used if access key / secret are unset. If not set the default will be used",
+        description="The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.",
     )
     aws_region: Optional[str] = Field(None, description="AWS region code.")
 
@@ -107,6 +291,14 @@ class AwsConnectionConfig(ConfigModel):
         default=None,
         description="A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.",
     )
+    aws_retry_num: int = Field(
+        default=5,
+        description="Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.",
+    )
+    aws_retry_mode: Literal["legacy", "standard", "adaptive"] = Field(
+        default="standard",
+        description="Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.",
+    )
 
     read_timeout: float = Field(
         default=DEFAULT_TIMEOUT,
@@ -137,6 +329,7 @@ def _normalized_aws_roles(self) -> List[AwsAssumeRoleConfig]:
 
     def get_session(self) -> Session:
         if self.aws_access_key_id and self.aws_secret_access_key:
+            # Explicit credentials take precedence
             session = Session(
                 aws_access_key_id=self.aws_access_key_id,
                 aws_secret_access_key=self.aws_secret_access_key,
@@ -144,38 +337,57 @@ def get_session(self) -> Session:
                 region_name=self.aws_region,
             )
         elif self.aws_profile:
+            # Named profile is second priority
             session = Session(
                 region_name=self.aws_region, profile_name=self.aws_profile
             )
         else:
-            # Use boto3's credential autodetection.
+            # Use boto3's credential autodetection
             session = Session(region_name=self.aws_region)
 
-        if self._normalized_aws_roles():
-            # Use existing session credentials to start the chain of role assumption.
-            current_credentials = session.get_credentials()
-            credentials = {
-                "AccessKeyId": current_credentials.access_key,
-                "SecretAccessKey": current_credentials.secret_key,
-                "SessionToken": current_credentials.token,
-            }
-
-            for role in self._normalized_aws_roles():
-                if self._should_refresh_credentials():
-                    credentials = assume_role(
-                        role,
-                        self.aws_region,
-                        credentials=credentials,
+            target_roles = self._normalized_aws_roles()
+            if target_roles:
+                current_role_arn, credential_source = get_current_identity()
+
+                # Only assume role if:
+                # 1. We're not in a known AWS environment with a role, or
+                # 2. We need to assume a different role than our current one
+                should_assume_role = current_role_arn is None or any(
+                    role.RoleArn != current_role_arn for role in target_roles
+                )
+
+                if should_assume_role:
+                    env = detect_aws_environment()
+                    logger.debug(f"Assuming role(s) from {env.value} environment")
+
+                    current_credentials = session.get_credentials()
+                    if current_credentials is None:
+                        raise ValueError("No credentials available for role assumption")
+
+                    credentials = {
+                        "AccessKeyId": current_credentials.access_key,
+                        "SecretAccessKey": current_credentials.secret_key,
+                        "SessionToken": current_credentials.token,
+                    }
+
+                    for role in target_roles:
+                        if self._should_refresh_credentials():
+                            credentials = assume_role(
+                                role=role,
+                                aws_region=self.aws_region,
+                                credentials=credentials,
+                            )
+                            if isinstance(credentials["Expiration"], datetime):
+                                self._credentials_expiration = credentials["Expiration"]
+
+                    session = Session(
+                        aws_access_key_id=credentials["AccessKeyId"],
+                        aws_secret_access_key=credentials["SecretAccessKey"],
+                        aws_session_token=credentials["SessionToken"],
+                        region_name=self.aws_region,
                     )
-                    if isinstance(credentials["Expiration"], datetime):
-                        self._credentials_expiration = credentials["Expiration"]
-
-            session = Session(
-                aws_access_key_id=credentials["AccessKeyId"],
-                aws_secret_access_key=credentials["SecretAccessKey"],
-                aws_session_token=credentials["SessionToken"],
-                region_name=self.aws_region,
-            )
+                else:
+                    logger.debug(f"Using existing role from {credential_source}")
 
         return session
 
@@ -199,6 +411,10 @@ def _aws_config(self) -> Config:
         return Config(
             proxies=self.aws_proxy,
             read_timeout=self.read_timeout,
+            retries={
+                "max_attempts": self.aws_retry_num,
+                "mode": self.aws_retry_mode,
+            },
             **self.aws_advanced_config,
         )
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
index 3b9b5dbf63e18..37c146218e263 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
@@ -678,12 +678,19 @@ def get_all_databases(self) -> Iterable[Mapping[str, Any]]:
         else:
             paginator_response = paginator.paginate()
 
-        for page in paginator_response:
-            yield from page["DatabaseList"]
+        pattern = "DatabaseList"
+        if self.source_config.ignore_resource_links:
+            # exclude resource links by using a JMESPath conditional query against the TargetDatabase struct key
+            pattern += "[?!TargetDatabase]"
+
+        for database in paginator_response.search(pattern):
+            if self.source_config.database_pattern.allowed(database["Name"]):
+                yield database
 
-    def get_tables_from_database(self, database_name: str) -> Iterable[Dict]:
+    def get_tables_from_database(self, database: Mapping[str, Any]) -> Iterable[Dict]:
         # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glue/paginator/GetTables.html
         paginator = self.glue_client.get_paginator("get_tables")
+        database_name = database["Name"]
 
         if self.source_config.catalog_id:
             paginator_response = paginator.paginate(
@@ -692,34 +699,28 @@ def get_tables_from_database(self, database_name: str) -> Iterable[Dict]:
         else:
             paginator_response = paginator.paginate(DatabaseName=database_name)
 
-        for page in paginator_response:
-            yield from page["TableList"]
+        for table in paginator_response.search("TableList"):
+            # if resource links are detected, re-use database names from the current catalog
+            # otherwise, external names are picked up instead of aliased ones when creating full table names later
+            # This will cause an incoherent situation when creating full table names later
+            # Note: use an explicit source_config check but it is useless actually (filtering has already been done)
+            if (
+                not self.source_config.ignore_resource_links
+                and "TargetDatabase" in database
+            ):
+                table["DatabaseName"] = database["Name"]
+            yield table
 
     def get_all_databases_and_tables(
         self,
-    ) -> Tuple[Dict, List[Dict]]:
-        all_databases = self.get_all_databases()
-
-        if self.source_config.ignore_resource_links:
-            all_databases = [
-                database
-                for database in all_databases
-                if "TargetDatabase" not in database
-            ]
-
-        allowed_databases = {
-            database["Name"]: database
-            for database in all_databases
-            if self.source_config.database_pattern.allowed(database["Name"])
-        }
-
+    ) -> Tuple[List[Mapping[str, Any]], List[Dict]]:
+        all_databases = [*self.get_all_databases()]
         all_tables = [
-            table
-            for database_name in allowed_databases
-            for table in self.get_tables_from_database(database_name)
+            tables
+            for database in all_databases
+            for tables in self.get_tables_from_database(database)
         ]
-
-        return allowed_databases, all_tables
+        return all_databases, all_tables
 
     def get_lineage_if_enabled(
         self, mce: MetadataChangeEventClass
@@ -1039,7 +1040,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         databases, tables = self.get_all_databases_and_tables()
 
-        for database in databases.values():
+        for database in databases:
             yield from self.gen_database_containers(database)
 
         for table in tables:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py
index b63fa57f069b5..55b8f4d889072 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py
@@ -1,3 +1,4 @@
+import logging
 from collections import defaultdict
 from typing import TYPE_CHECKING, DefaultDict, Dict, Iterable, List, Optional
 
@@ -36,6 +37,8 @@
 if TYPE_CHECKING:
     from mypy_boto3_sagemaker import SageMakerClient
 
+logger = logging.getLogger(__name__)
+
 
 @platform_name("SageMaker")
 @config_class(SagemakerSourceConfig)
@@ -75,6 +78,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         ]
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        logger.info("Starting SageMaker ingestion...")
         # get common lineage graph
         lineage_processor = LineageProcessor(
             sagemaker_client=self.sagemaker_client, env=self.env, report=self.report
@@ -83,6 +87,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
         # extract feature groups if specified
         if self.source_config.extract_feature_groups:
+            logger.info("Extracting feature groups...")
             feature_group_processor = FeatureGroupProcessor(
                 sagemaker_client=self.sagemaker_client, env=self.env, report=self.report
             )
@@ -95,6 +100,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
         # extract jobs if specified
         if self.source_config.extract_jobs is not False:
+            logger.info("Extracting jobs...")
             job_processor = JobProcessor(
                 sagemaker_client=self.client_factory.get_client,
                 env=self.env,
@@ -109,6 +115,8 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
         # extract models if specified
         if self.source_config.extract_models:
+            logger.info("Extracting models...")
+
             model_processor = ModelProcessor(
                 sagemaker_client=self.sagemaker_client,
                 env=self.env,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/common.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/common.py
index 45dadab7c24df..73d8d33dd11be 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/common.py
@@ -40,8 +40,11 @@ class SagemakerSourceReport(StaleEntityRemovalSourceReport):
     groups_scanned = 0
     models_scanned = 0
     jobs_scanned = 0
+    jobs_processed = 0
     datasets_scanned = 0
     filtered: List[str] = field(default_factory=list)
+    model_endpoint_lineage = 0
+    model_group_lineage = 0
 
     def report_feature_group_scanned(self) -> None:
         self.feature_groups_scanned += 1
@@ -58,6 +61,9 @@ def report_group_scanned(self) -> None:
     def report_model_scanned(self) -> None:
         self.models_scanned += 1
 
+    def report_job_processed(self) -> None:
+        self.jobs_processed += 1
+
     def report_job_scanned(self) -> None:
         self.jobs_scanned += 1
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py
index b8b96c6306a3b..c4561b9d9e676 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py
@@ -1,3 +1,5 @@
+import logging
+import textwrap
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Iterable, List
 
@@ -28,6 +30,8 @@
         FeatureGroupSummaryTypeDef,
     )
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class FeatureGroupProcessor:
@@ -197,11 +201,12 @@ def get_feature_wu(
 
                 full_table_name = f"{glue_database}.{glue_table}"
 
-                self.report.report_warning(
-                    full_table_name,
-                    f"""Note: table {full_table_name} is an AWS Glue object.
+                logging.info(
+                    textwrap.dedent(
+                        f"""Note: table {full_table_name} is an AWS Glue object. This source does not ingest all metadata for Glue tables.
                         To view full table metadata, run Glue ingestion
-                        (see https://datahubproject.io/docs/metadata-ingestion/#aws-glue-glue)""",
+                        (see https://datahubproject.io/docs/generated/ingestion/sources/glue)"""
+                    )
                 )
 
                 feature_sources.append(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py
index 73a83295ec8cb..be0a99c6d3234 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py
@@ -1,3 +1,4 @@
+import logging
 from collections import defaultdict
 from dataclasses import dataclass, field
 from enum import Enum
@@ -49,6 +50,8 @@
 if TYPE_CHECKING:
     from mypy_boto3_sagemaker import SageMakerClient
 
+logger = logging.getLogger(__name__)
+
 JobInfo = TypeVar(
     "JobInfo",
     AutoMlJobInfo,
@@ -274,15 +277,18 @@ def get_job_details(self, job_name: str, job_type: JobType) -> Dict[str, Any]:
         )
 
     def get_workunits(self) -> Iterable[MetadataWorkUnit]:
+        logger.info("Getting all SageMaker jobs")
         jobs = self.get_all_jobs()
 
         processed_jobs: Dict[str, SageMakerJob] = {}
 
+        logger.info("Processing SageMaker jobs")
         # first pass: process jobs and collect datasets used
+        logger.info("first pass: process jobs and collect datasets used")
         for job in jobs:
             job_type = job_type_to_info[job["type"]]
             job_name = job[job_type.list_name_key]
-
+            logger.debug(f"Processing job {job_name} with type {job_type}")
             job_details = self.get_job_details(job_name, job["type"])
 
             processed_job = getattr(self, job_type.processor)(job_details)
@@ -293,6 +299,9 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
         # second pass:
         #   - move output jobs to inputs
         #   - aggregate i/o datasets
+        logger.info(
+            "second pass: move output jobs to inputs and aggregate i/o datasets"
+        )
         for job_urn in sorted(processed_jobs):
             processed_job = processed_jobs[job_urn]
 
@@ -301,6 +310,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
 
             all_datasets.update(processed_job.input_datasets)
             all_datasets.update(processed_job.output_datasets)
+            self.report.report_job_processed()
 
         # yield datasets
         for dataset_urn, dataset in all_datasets.items():
@@ -322,6 +332,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
             self.report.report_dataset_scanned()
 
         # third pass: construct and yield MCEs
+        logger.info("third pass: construct and yield MCEs")
         for job_urn in sorted(processed_jobs):
             processed_job = processed_jobs[job_urn]
             job_snapshot = processed_job.job_snapshot
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py
index b677dccad24ac..24e5497269c73 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py
@@ -1,3 +1,4 @@
+import logging
 from collections import defaultdict
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, DefaultDict, Dict, List, Set
@@ -6,6 +7,8 @@
     SagemakerSourceReport,
 )
 
+logger = logging.getLogger(__name__)
+
 if TYPE_CHECKING:
     from mypy_boto3_sagemaker import SageMakerClient
     from mypy_boto3_sagemaker.type_defs import (
@@ -88,7 +91,6 @@ def get_all_contexts(self) -> List["ContextSummaryTypeDef"]:
         paginator = self.sagemaker_client.get_paginator("list_contexts")
         for page in paginator.paginate():
             contexts += page["ContextSummaries"]
-
         return contexts
 
     def get_incoming_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]:
@@ -225,27 +227,32 @@ def get_lineage(self) -> LineageInfo:
         """
         Get the lineage of all artifacts in SageMaker.
         """
-
+        logger.info("Getting lineage for SageMaker artifacts...")
+        logger.info("Getting all actions")
         for action in self.get_all_actions():
             self.nodes[action["ActionArn"]] = {**action, "node_type": "action"}
+        logger.info("Getting all artifacts")
         for artifact in self.get_all_artifacts():
             self.nodes[artifact["ArtifactArn"]] = {**artifact, "node_type": "artifact"}
+        logger.info("Getting all contexts")
         for context in self.get_all_contexts():
             self.nodes[context["ContextArn"]] = {**context, "node_type": "context"}
 
+        logger.info("Getting lineage for model deployments and model groups")
         for node_arn, node in self.nodes.items():
+            logger.debug(f"Getting lineage for node {node_arn}")
             # get model-endpoint lineage
             if (
                 node["node_type"] == "action"
                 and node.get("ActionType") == "ModelDeployment"
             ):
                 self.get_model_deployment_lineage(node_arn)
-
+                self.report.model_endpoint_lineage += 1
             # get model-group lineage
             if (
                 node["node_type"] == "context"
                 and node.get("ContextType") == "ModelGroup"
             ):
                 self.get_model_group_lineage(node_arn, node)
-
+                self.report.model_group_lineage += 1
         return self.lineage_info
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py
index eef2b26ee08f2..0f433aaecf2d9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py
@@ -1,3 +1,4 @@
+import logging
 from collections import defaultdict
 from dataclasses import dataclass, field
 from datetime import datetime
@@ -65,6 +66,8 @@
     "Unknown": DeploymentStatusClass.UNKNOWN,
 }
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class ModelProcessor:
@@ -385,6 +388,26 @@ def strip_quotes(string: str) -> str:
             model_metrics,
         )
 
+    @staticmethod
+    def get_group_name_from_arn(arn: str) -> str:
+        """
+        Extract model package group name from a SageMaker ARN.
+
+        Args:
+            arn (str): Full ARN of the model package group
+
+        Returns:
+            str: Name of the model package group
+
+        Example:
+            >>> ModelProcessor.get_group_name_from_arn("arn:aws:sagemaker:eu-west-1:123456789:model-package-group/my-model-group")
+            'my-model-group'
+        """
+        logger.debug(
+            f"Extracting group name from ARN: {arn} because group was not seen before"
+        )
+        return arn.split("/")[-1]
+
     def get_model_wu(
         self,
         model_details: "DescribeModelOutputTypeDef",
@@ -425,8 +448,14 @@ def get_model_wu(
         model_group_arns = model_uri_groups | model_image_groups
 
         model_group_names = sorted(
-            [self.group_arn_to_name[x] for x in model_group_arns]
+            [
+                self.group_arn_to_name[x]
+                if x in self.group_arn_to_name
+                else self.get_group_name_from_arn(x)
+                for x in model_group_arns
+            ]
         )
+
         model_group_urns = [
             builder.make_ml_model_group_urn("sagemaker", x, self.env)
             for x in model_group_names
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
index 4cc3ec50bacd4..16a5268a2dea7 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -65,9 +65,6 @@
 # We can't use close as it is not called if the ingestion is not successful
 def cleanup(config: BigQueryV2Config) -> None:
     if config._credentials_path is not None:
-        logger.debug(
-            f"Deleting temporary credential file at {config._credentials_path}"
-        )
         os.unlink(config._credentials_path)
 
 
@@ -98,6 +95,10 @@ def cleanup(config: BigQueryV2Config) -> None:
     "Optionally enabled via `classification.enabled`",
     supported=True,
 )
+@capability(
+    SourceCapability.PARTITION_SUPPORT,
+    "Enabled by default, partition keys and clustering keys are supported.",
+)
 class BigqueryV2Source(StatefulIngestionSourceBase, TestableSource):
     def __init__(self, ctx: PipelineContext, config: BigQueryV2Config):
         super().__init__(config, ctx)
@@ -188,6 +189,7 @@ def __init__(self, ctx: PipelineContext, config: BigQueryV2Config):
             self.sql_parser_schema_resolver,
             self.profiler,
             self.identifiers,
+            self.ctx.graph,
         )
 
         self.add_config_to_report()
@@ -274,7 +276,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
             self.report.set_ingestion_stage("*", QUERIES_EXTRACTION)
 
-            queries_extractor = BigQueryQueriesExtractor(
+            with BigQueryQueriesExtractor(
                 connection=self.config.get_bigquery_client(),
                 schema_api=self.bq_schema_extractor.schema_api,
                 config=BigQueryQueriesExtractorConfig(
@@ -284,15 +286,17 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     include_usage_statistics=self.config.include_usage_statistics,
                     include_operations=self.config.usage.include_operational_stats,
                     top_n_queries=self.config.usage.top_n_queries,
+                    region_qualifiers=self.config.region_qualifiers,
                 ),
                 structured_report=self.report,
                 filters=self.filters,
                 identifiers=self.identifiers,
                 schema_resolver=self.sql_parser_schema_resolver,
                 discovered_tables=self.bq_schema_extractor.table_refs,
-            )
-            self.report.queries_extractor = queries_extractor.report
-            yield from queries_extractor.get_workunits_internal()
+            ) as queries_extractor:
+                self.report.queries_extractor = queries_extractor.report
+                yield from queries_extractor.get_workunits_internal()
+
         else:
             if self.config.include_usage_statistics:
                 yield from self.usage_extractor.get_usage_workunits(
@@ -309,6 +313,16 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     self.bq_schema_extractor.table_refs,
                 )
 
+        # Lineage BQ to GCS
+        if (
+            self.config.include_table_lineage
+            and self.bq_schema_extractor.external_tables
+        ):
+            for dataset_urn, table in self.bq_schema_extractor.external_tables.items():
+                yield from self.lineage_extractor.gen_lineage_workunits_for_external_table(
+                    dataset_urn, table.ddl, graph=self.ctx.graph
+                )
+
     def get_report(self) -> BigQueryV2Report:
         return self.report
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
index 319c838d2658a..42f82704c81b9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
@@ -190,7 +190,7 @@ def from_string_name(cls, ref: str) -> "BigQueryTableRef":
     @classmethod
     def from_urn(cls, urn: str) -> "BigQueryTableRef":
         """Raises: ValueError if urn is not a valid BigQuery table URN."""
-        dataset_urn = DatasetUrn.create_from_string(urn)
+        dataset_urn = DatasetUrn.from_string(urn)
         split = dataset_urn.name.rsplit(".", 3)
         if len(split) == 3:
             project, dataset, table = split
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
index 2a34fb82c2050..4af41921c9fa3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
@@ -21,6 +21,7 @@
 from datahub.ingestion.glossary.classification_mixin import (
     ClassificationSourceConfigMixin,
 )
+from datahub.ingestion.source.data_lake_common.path_spec import PathSpec
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig, SQLFilterConfig
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulLineageConfigMixin,
@@ -206,6 +207,39 @@ def get_sql_alchemy_url(self) -> str:
         return "bigquery://"
 
 
+class GcsLineageProviderConfig(ConfigModel):
+    """
+    Any source that produces gcs lineage from/to Datasets should inherit this class.
+    """
+
+    path_specs: List[PathSpec] = Field(
+        default=[],
+        description="List of PathSpec. See below the details about PathSpec",
+    )
+
+    strip_urls: bool = Field(
+        default=True,
+        description="Strip filename from gcs url. It only applies if path_specs are not specified.",
+    )
+
+    ignore_non_path_spec_path: bool = Field(
+        default=False,
+        description="Ignore paths that are not match in path_specs. It only applies if path_specs are specified.",
+    )
+
+
+class GcsDatasetLineageProviderConfigBase(ConfigModel):
+    """
+    Any source that produces gcs lineage from/to Datasets should inherit this class.
+    This is needeed to group all lineage related configs under `gcs_lineage_config` config property.
+    """
+
+    gcs_lineage_config: GcsLineageProviderConfig = Field(
+        default=GcsLineageProviderConfig(),
+        description="Common config for gcs lineage generation",
+    )
+
+
 class BigQueryFilterConfig(SQLFilterConfig):
     project_ids: List[str] = Field(
         default_factory=list,
@@ -328,6 +362,7 @@ class BigQueryIdentifierConfig(
 
 
 class BigQueryV2Config(
+    GcsDatasetLineageProviderConfigBase,
     BigQueryConnectionConfig,
     BigQueryBaseConfig,
     BigQueryFilterConfig,
@@ -339,7 +374,6 @@ class BigQueryV2Config(
     StatefulProfilingConfigMixin,
     ClassificationSourceConfigMixin,
 ):
-
     include_schema_metadata: bool = Field(
         default=True,
         description="Whether to ingest the BigQuery schema, i.e. projects, schemas, tables, and views.",
@@ -369,6 +403,11 @@ class BigQueryV2Config(
         description="Capture BigQuery dataset labels as DataHub tag",
     )
 
+    include_table_constraints: bool = Field(
+        default=True,
+        description="Whether to ingest table constraints. If you know you don't use table constraints, you can disable it to save one extra query per dataset. In general it should be enabled",
+    )
+
     include_external_url: bool = Field(
         default=True,
         description="Whether to populate BigQuery Console url to Datasets/Tables",
@@ -468,6 +507,11 @@ def have_table_data_read_permission(self) -> bool:
         description="Option to enable/disable lineage generation. Is enabled by default.",
     )
 
+    include_column_lineage_with_gcs: bool = Field(
+        default=True,
+        description="When enabled, column-level lineage will be extracted from the gcs.",
+    )
+
     max_query_duration: timedelta = Field(
         default=timedelta(minutes=15),
         description="Correction to pad start_time and end_time with. For handling the case where the read happens within our time range but the query completion event is delayed and happens after the configured end time.",
@@ -522,6 +566,12 @@ def have_table_data_read_permission(self) -> bool:
         " Set to 1 to disable.",
     )
 
+    region_qualifiers: List[str] = Field(
+        default=["region-us", "region-eu"],
+        description="BigQuery regions to be scanned for bigquery jobs when using `use_queries_v2`. "
+        "See [this](https://cloud.google.com/bigquery/docs/information-schema-jobs#scope_and_syntax) for details.",
+    )
+
     # include_view_lineage and include_view_column_lineage are inherited from SQLCommonConfig
     # but not used in bigquery so we hide them from docs.
     include_view_lineage: bool = Field(default=True, hidden_from_docs=True)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py
new file mode 100644
index 0000000000000..9da2aceb19220
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py
@@ -0,0 +1,144 @@
+import logging
+from dataclasses import dataclass
+from typing import Optional
+
+import cachetools
+from pydantic import BaseModel, ValidationError
+
+from datahub.api.entities.platformresource.platform_resource import (
+    PlatformResource,
+    PlatformResourceKey,
+)
+from datahub.ingestion.graph.client import DataHubGraph
+from datahub.metadata.urns import TagUrn
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+@dataclass
+class BigQueryLabel:
+    key: str
+    value: Optional[str]
+
+    def primary_key(self) -> str:
+        return f"{self.key}/{self.value}" if self.value else f"{self.key}"
+
+
+class BigQueryLabelInfo(BaseModel):
+    datahub_urn: str
+    managed_by_datahub: bool
+    key: str
+    value: str
+
+
+@dataclass()
+class BigQueryLabelPlatformResource:
+    datahub_urn: str
+    project: Optional[str]
+    managed_by_datahub: bool
+    label: BigQueryLabel
+
+    def platform_resource_key(self) -> PlatformResourceKey:
+        return PlatformResourceKey(
+            platform="bigquery",
+            resource_type="BigQueryLabelInfo",
+            platform_instance=None,
+            primary_key=self.label.primary_key(),
+        )
+
+    def platform_resource_info(self) -> BigQueryLabelInfo:
+        bq_label_info = BigQueryLabelInfo(
+            datahub_urn=self.datahub_urn,
+            managed_by_datahub=self.managed_by_datahub,
+            key=self.label.key,
+            value=self.label.value,
+        )
+        return bq_label_info
+
+    def platform_resource(self) -> PlatformResource:
+        return PlatformResource.create(
+            key=self.platform_resource_key(),
+            secondary_keys=[self.datahub_urn],
+            value=self.platform_resource_info(),
+        )
+
+
+class BigQueryPlatformResourceHelper:
+    def __init__(
+        self,
+        bq_project: Optional[str],
+        graph: Optional[DataHubGraph],
+    ):
+        self.bq_project = bq_project
+        self.graph = graph
+
+    platform_resource_cache: cachetools.LRUCache = cachetools.LRUCache(maxsize=500)
+
+    def get_platform_resource(
+        self, platform_resource_key: PlatformResourceKey
+    ) -> Optional[PlatformResource]:
+        # if graph is not available we always create a new PlatformResource
+        if not self.graph:
+            return None
+        if self.platform_resource_cache.get(platform_resource_key.primary_key):
+            return self.platform_resource_cache.get(platform_resource_key.primary_key)
+
+        platform_resource = PlatformResource.from_datahub(
+            key=platform_resource_key, graph_client=self.graph
+        )
+        if platform_resource:
+            self.platform_resource_cache[
+                platform_resource_key.primary_key
+            ] = platform_resource
+            return platform_resource
+        return None
+
+    def generate_label_platform_resource(
+        self,
+        bigquery_label: BigQueryLabel,
+        tag_urn: TagUrn,
+        managed_by_datahub: bool = True,
+    ) -> PlatformResource:
+        new_platform_resource = BigQueryLabelPlatformResource(
+            datahub_urn=tag_urn.urn(),
+            project=self.bq_project,
+            managed_by_datahub=managed_by_datahub,
+            label=bigquery_label,
+        )
+
+        platform_resource = self.get_platform_resource(
+            new_platform_resource.platform_resource_key()
+        )
+        if platform_resource:
+            if (
+                platform_resource.resource_info
+                and platform_resource.resource_info.value
+            ):
+                try:
+                    existing_info: Optional[BigQueryLabelInfo] = platform_resource.resource_info.value.as_pydantic_object(BigQueryLabelInfo)  # type: ignore
+                except ValidationError as e:
+                    logger.error(
+                        f"Error converting existing value to BigQueryLabelInfo: {e}. Creating new one. Maybe this is because of a non backward compatible schema change."
+                    )
+                    existing_info = None
+
+                if existing_info:
+                    if (
+                        new_platform_resource.platform_resource_info() == existing_info
+                        or existing_info.managed_by_datahub
+                    ):
+                        return platform_resource
+                    else:
+                        raise ValueError(
+                            f"Datahub URN mismatch for platform resources. Old (existing) platform resource: {platform_resource} and new platform resource: {new_platform_resource}"
+                        )
+
+        logger.info(f"Created platform resource {new_platform_resource}")
+
+        self.platform_resource_cache.update(
+            {
+                new_platform_resource.platform_resource_key().primary_key: new_platform_resource.platform_resource()
+            }
+        )
+
+        return new_platform_resource.platform_resource()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_queries.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_queries.py
index ed27aae19ce96..47f21c9f32353 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_queries.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_queries.py
@@ -88,3 +88,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
     def get_report(self) -> BigQueryQueriesSourceReport:
         return self.report
+
+    def close(self) -> None:
+        self.queries_extractor.close()
+        self.connection.close()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py
index cc8c6482eb9ef..06842da67f76c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py
@@ -8,7 +8,7 @@
 
 from datahub.ingestion.api.report import Report
 from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin
-from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.ingestion.source_report.time_window import BaseTimeWindowReport
 from datahub.sql_parsing.sql_parsing_aggregator import SqlAggregatorReport
@@ -30,6 +30,7 @@ class BigQuerySchemaApiPerfReport(Report):
     num_list_tables_api_requests: int = 0
     num_get_views_for_dataset_api_requests: int = 0
     num_get_snapshots_for_dataset_api_requests: int = 0
+    num_get_table_constraints_for_dataset_api_requests: int = 0
 
     list_projects_timer: PerfTimer = field(default_factory=PerfTimer)
     list_projects_with_labels_timer: PerfTimer = field(default_factory=PerfTimer)
@@ -37,6 +38,7 @@ class BigQuerySchemaApiPerfReport(Report):
 
     get_columns_for_dataset_sec: float = 0
     get_tables_for_dataset_sec: float = 0
+    get_table_constraints_for_dataset_sec: float = 0
     list_tables_sec: float = 0
     get_views_for_dataset_sec: float = 0
     get_snapshots_for_dataset_sec: float = 0
@@ -75,7 +77,7 @@ class BigQueryQueriesExtractorReport(Report):
 
 @dataclass
 class BigQueryV2Report(
-    ProfilingSqlReport,
+    SQLSourceReport,
     IngestionStageReport,
     BaseTimeWindowReport,
     ClassificationReportMixin,
@@ -155,6 +157,7 @@ class BigQueryV2Report(
     num_filtered_query_events: int = 0
     num_usage_query_hash_collisions: int = 0
     num_operational_stats_workunits_emitted: int = 0
+    num_lineage_dropped_gcs_path: int = 0
 
     snapshots_scanned: int = 0
 
@@ -184,6 +187,7 @@ class BigQueryV2Report(
     usage_start_time: Optional[datetime] = None
     usage_end_time: Optional[datetime] = None
     stateful_usage_ingestion_enabled: bool = False
+    num_skipped_external_table_lineage: int = 0
 
     queries_extractor: Optional[BigQueryQueriesExtractorReport] = None
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
index dd683559a007b..3ce34be8dc89d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
@@ -42,6 +42,20 @@ class BigqueryColumn(BaseColumn):
     policy_tags: Optional[List[str]] = None
 
 
+@dataclass
+class BigqueryTableConstraint:
+    name: str
+    project_id: str
+    dataset_name: str
+    table_name: str
+    type: str
+    field_path: str
+    referenced_project_id: Optional[str] = None
+    referenced_dataset: Optional[str] = None
+    referenced_table_name: Optional[str] = None
+    referenced_column_name: Optional[str] = None
+
+
 RANGE_PARTITION_NAME: str = "RANGE"
 
 
@@ -104,8 +118,9 @@ class BigqueryTable(BaseTable):
     active_billable_bytes: Optional[int] = None
     long_term_billable_bytes: Optional[int] = None
     partition_info: Optional[PartitionInfo] = None
-    columns_ignore_from_profiling: List[str] = field(default_factory=list)
     external: bool = False
+    constraints: List[BigqueryTableConstraint] = field(default_factory=list)
+    table_type: Optional[str] = None
 
 
 @dataclass
@@ -136,6 +151,21 @@ class BigqueryDataset:
     snapshots: List[BigqueryTableSnapshot] = field(default_factory=list)
     columns: List[BigqueryColumn] = field(default_factory=list)
 
+    # Some INFORMATION_SCHEMA views are not available for BigLake tables
+    # based on Amazon S3 and Blob Storage data.
+    # https://cloud.google.com/bigquery/docs/omni-introduction#limitations
+    # Omni Locations - https://cloud.google.com/bigquery/docs/omni-introduction#locations
+    def is_biglake_dataset(self) -> bool:
+        return self.location is not None and self.location.lower().startswith(
+            ("aws-", "azure-")
+        )
+
+    def supports_table_constraints(self) -> bool:
+        return not self.is_biglake_dataset()
+
+    def supports_table_partitions(self) -> bool:
+        return not self.is_biglake_dataset()
+
 
 @dataclass
 class BigqueryProject:
@@ -159,7 +189,7 @@ def __init__(
 
     def get_query_result(self, query: str) -> RowIterator:
         def _should_retry(exc: BaseException) -> bool:
-            logger.debug(f"Exception occured for job query. Reason: {exc}")
+            logger.debug(f"Exception occurred for job query. Reason: {exc}")
             # Jobs sometimes fail with transient errors.
             # This is not currently handled by the python-bigquery client.
             # https://github.com/googleapis/python-bigquery/issues/23
@@ -181,7 +211,7 @@ def _should_retry(exc: BaseException) -> bool:
     def get_projects(self, max_results_per_page: int = 100) -> List[BigqueryProject]:
         def _should_retry(exc: BaseException) -> bool:
             logger.debug(
-                f"Exception occured for project.list api. Reason: {exc}. Retrying api request..."
+                f"Exception occurred for project.list api. Reason: {exc}. Retrying api request..."
             )
             self.report.num_list_projects_retry_request += 1
             return True
@@ -362,6 +392,7 @@ def _make_bigquery_table(
         return BigqueryTable(
             name=table.table_name,
             created=table.created,
+            table_type=table.table_type,
             last_altered=(
                 datetime.fromtimestamp(
                     table.get("last_altered") / 1000, tz=timezone.utc
@@ -492,6 +523,65 @@ def get_policy_tags_for_column(
                 exc=e,
             )
 
+    def get_table_constraints_for_dataset(
+        self,
+        project_id: str,
+        dataset_name: str,
+        report: BigQueryV2Report,
+    ) -> Optional[Dict[str, List[BigqueryTableConstraint]]]:
+        constraints: Dict[str, List[BigqueryTableConstraint]] = defaultdict(list)
+        with PerfTimer() as timer:
+            try:
+                cur = self.get_query_result(
+                    BigqueryQuery.constraints_for_table.format(
+                        project_id=project_id, dataset_name=dataset_name
+                    )
+                )
+            except Exception as e:
+                report.warning(
+                    title="Failed to retrieve table constraints for dataset",
+                    message="Query to get table constraints for dataset failed with exception",
+                    context=f"{project_id}.{dataset_name}",
+                    exc=e,
+                )
+                return None
+
+            for constraint in cur:
+                constraints[constraint.table_name].append(
+                    BigqueryTableConstraint(
+                        name=constraint.constraint_name,
+                        project_id=constraint.table_catalog,
+                        dataset_name=constraint.table_schema,
+                        table_name=constraint.table_name,
+                        type=constraint.constraint_type,
+                        field_path=constraint.column_name,
+                        referenced_project_id=(
+                            constraint.referenced_catalog
+                            if constraint.constraint_type == "FOREIGN KEY"
+                            else None
+                        ),
+                        referenced_dataset=(
+                            constraint.referenced_schema
+                            if constraint.constraint_type == "FOREIGN KEY"
+                            else None
+                        ),
+                        referenced_table_name=(
+                            constraint.referenced_table
+                            if constraint.constraint_type == "FOREIGN KEY"
+                            else None
+                        ),
+                        referenced_column_name=(
+                            constraint.referenced_column
+                            if constraint.constraint_type == "FOREIGN KEY"
+                            else None
+                        ),
+                    )
+                )
+            self.report.num_get_table_constraints_for_dataset_api_requests += 1
+            self.report.get_table_constraints_for_dataset_sec += timer.elapsed_seconds()
+
+        return constraints
+
     def get_columns_for_dataset(
         self,
         project_id: str,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py
index 11d06771d4e4f..4a3b47f6b543a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py
@@ -2,12 +2,18 @@
 import re
 from base64 import b32decode
 from collections import defaultdict
+from itertools import groupby
 from typing import Dict, Iterable, List, Optional, Set, Type, Union, cast
 
 from google.cloud.bigquery.table import TableListItem
 
+from datahub.api.entities.platformresource.platform_resource import PlatformResource
 from datahub.configuration.pattern_utils import is_schema_allowed, is_tag_allowed
-from datahub.emitter.mce_builder import make_tag_urn
+from datahub.emitter.mce_builder import (
+    make_dataset_urn_with_platform_instance,
+    make_schema_field_urn,
+    make_tag_urn,
+)
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import BigQueryDatasetKey, ContainerKey, ProjectIdKey
 from datahub.ingestion.api.workunit import MetadataWorkUnit
@@ -16,6 +22,7 @@
     ClassificationHandler,
     classification_workunit_processor,
 )
+from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import (
     BigqueryTableIdentifier,
     BigQueryTableRef,
@@ -25,6 +32,11 @@
 from datahub.ingestion.source.bigquery_v2.bigquery_helper import (
     unquote_and_decode_unicode_escape_seq,
 )
+from datahub.ingestion.source.bigquery_v2.bigquery_platform_resource_helper import (
+    BigQueryLabel,
+    BigQueryLabelInfo,
+    BigQueryPlatformResourceHelper,
+)
 from datahub.ingestion.source.bigquery_v2.bigquery_report import BigQueryV2Report
 from datahub.ingestion.source.bigquery_v2.bigquery_schema import (
     BigqueryColumn,
@@ -32,6 +44,7 @@
     BigqueryProject,
     BigQuerySchemaApi,
     BigqueryTable,
+    BigqueryTableConstraint,
     BigqueryTableSnapshot,
     BigqueryView,
 )
@@ -69,6 +82,7 @@
     BooleanType,
     BytesType,
     DateType,
+    ForeignKeyConstraint,
     MySqlDDL,
     NullType,
     NumberType,
@@ -84,6 +98,7 @@
     GlobalTagsClass,
     TagAssociationClass,
 )
+from datahub.metadata.urns import TagUrn
 from datahub.sql_parsing.schema_resolver import SchemaResolver
 from datahub.utilities.file_backed_collections import FileBackedDict
 from datahub.utilities.hive_schema_to_avro import (
@@ -160,6 +175,7 @@ def __init__(
         sql_parser_schema_resolver: SchemaResolver,
         profiler: BigqueryProfiler,
         identifiers: BigQueryIdentifierBuilder,
+        graph: Optional[DataHubGraph] = None,
     ):
         self.config = config
         self.report = report
@@ -168,6 +184,7 @@ def __init__(
         self.sql_parser_schema_resolver = sql_parser_schema_resolver
         self.profiler = profiler
         self.identifiers = identifiers
+        self.graph = graph
 
         self.classification_handler = ClassificationHandler(self.config, self.report)
         self.data_reader: Optional[BigQueryDataReader] = None
@@ -187,6 +204,26 @@ def __init__(
         self.view_definitions: FileBackedDict[str] = FileBackedDict()
         # Maps snapshot ref -> Snapshot
         self.snapshots_by_ref: FileBackedDict[BigqueryTableSnapshot] = FileBackedDict()
+        # Add External BQ table
+        self.external_tables: Dict[str, BigqueryTable] = defaultdict()
+        self.bq_external_table_pattern = (
+            r".*create\s+external\s+table\s+`?(?:project_id\.)?.*`?"
+        )
+
+        bq_project = (
+            self.config.project_on_behalf
+            if self.config.project_on_behalf
+            else self.config.credential.project_id
+            if self.config.credential
+            else None
+        )
+
+        self.platform_resource_helper: BigQueryPlatformResourceHelper = (
+            BigQueryPlatformResourceHelper(
+                bq_project,
+                self.graph,
+            )
+        )
 
     @property
     def store_table_refs(self):
@@ -264,13 +301,28 @@ def gen_dataset_containers(
     ) -> Iterable[MetadataWorkUnit]:
         schema_container_key = self.gen_dataset_key(project_id, dataset)
 
-        tags_joined: Optional[List[str]] = None
+        tags_joined: List[str] = []
         if tags and self.config.capture_dataset_label_as_tag:
-            tags_joined = [
-                self.make_tag_from_label(k, v)
-                for k, v in tags.items()
-                if is_tag_allowed(self.config.capture_dataset_label_as_tag, k)
-            ]
+            for k, v in tags.items():
+                if is_tag_allowed(self.config.capture_dataset_label_as_tag, k):
+                    tag_urn = TagUrn.from_string(self.make_tag_urn_from_label(k, v))
+                    label = BigQueryLabel(key=k, value=v)
+                    try:
+                        platform_resource: PlatformResource = self.platform_resource_helper.generate_label_platform_resource(
+                            label, tag_urn, managed_by_datahub=False
+                        )
+                        label_info: BigQueryLabelInfo = platform_resource.resource_info.value.as_pydantic_object(  # type: ignore
+                            BigQueryLabelInfo
+                        )
+                        tag_urn = TagUrn.from_string(label_info.datahub_urn)
+
+                        for mcpw in platform_resource.to_mcps():
+                            yield mcpw.as_workunit()
+                    except ValueError as e:
+                        logger.warning(
+                            f"Failed to generate platform resource for label {k}:{v}: {e}"
+                        )
+                    tags_joined.append(tag_urn.name)
 
         database_container_key = self.gen_project_id_key(database=project_id)
 
@@ -304,7 +356,6 @@ def _process_project(
                 project_id
             )
         except Exception as e:
-
             if self.config.project_ids and "not enabled BigQuery." in str(e):
                 action_mesage = (
                     "The project has not enabled BigQuery API. "
@@ -365,7 +416,6 @@ def _process_project_datasets(
         bigquery_project: BigqueryProject,
         db_tables: Dict[str, List[BigqueryTable]],
     ) -> Iterable[MetadataWorkUnit]:
-
         db_views: Dict[str, List[BigqueryView]] = {}
         db_snapshots: Dict[str, List[BigqueryTableSnapshot]] = {}
         project_id = bigquery_project.id
@@ -430,6 +480,7 @@ def _process_schema(
             )
 
         columns = None
+        constraints: Optional[Dict[str, List[BigqueryTableConstraint]]] = None
 
         rate_limiter: Optional[RateLimiter] = None
         if self.config.rate_limit:
@@ -447,6 +498,13 @@ def _process_schema(
                 report=self.report,
                 rate_limiter=rate_limiter,
             )
+            if (
+                self.config.include_table_constraints
+                and bigquery_dataset.supports_table_constraints()
+            ):
+                constraints = self.schema_api.get_table_constraints_for_dataset(
+                    project_id=project_id, dataset_name=dataset_name, report=self.report
+                )
         elif self.store_table_refs:
             # Need table_refs to calculate lineage and usage
             for table_item in self.schema_api.list_tables(dataset_name, project_id):
@@ -466,7 +524,6 @@ def _process_schema(
                     logger.warning(
                         f"Could not create table ref for {table_item.path}: {e}"
                     )
-            yield from []
             return
 
         if self.config.include_tables:
@@ -476,6 +533,11 @@ def _process_schema(
 
             for table in db_tables[dataset_name]:
                 table_columns = columns.get(table.name, []) if columns else []
+                table_constraints = (
+                    constraints.get(table.name, []) if constraints else []
+                )
+
+                table.constraints = table_constraints
                 table_wu_generator = self._process_table(
                     table=table,
                     columns=table_columns,
@@ -536,18 +598,6 @@ def _process_schema(
                     dataset_name=dataset_name,
                 )
 
-    # This method is used to generate the ignore list for datatypes the profiler doesn't support we have to do it here
-    # because the profiler doesn't have access to columns
-    def generate_profile_ignore_list(self, columns: List[BigqueryColumn]) -> List[str]:
-        ignore_list: List[str] = []
-        for column in columns:
-            if not column.data_type or any(
-                word in column.data_type.lower()
-                for word in ["array", "struct", "geography", "json"]
-            ):
-                ignore_list.append(column.field_path)
-        return ignore_list
-
     def _process_table(
         self,
         table: BigqueryTable,
@@ -569,15 +619,6 @@ def _process_table(
             )
         table.column_count = len(columns)
 
-        # We only collect profile ignore list if profiling is enabled and profile_table_level_only is false
-        if (
-            self.config.is_profiling_enabled()
-            and not self.config.profiling.profile_table_level_only
-        ):
-            table.columns_ignore_from_profiling = self.generate_profile_ignore_list(
-                columns
-            )
-
         if not table.column_count:
             logger.warning(
                 f"Table doesn't have any column or unable to get columns for table: {table_identifier}"
@@ -676,10 +717,63 @@ def _process_snapshot(
             dataset_name=dataset_name,
         )
 
-    def make_tag_from_label(self, key: str, value: str) -> str:
-        if not value.startswith(ENCODED_TAG_PREFIX):
+    def make_tag_urn_from_label(self, key: str, value: str) -> str:
+        if value:
             return make_tag_urn(f"""{key}:{value}""")
-        return self.modified_base32decode(value)
+        else:
+            return make_tag_urn(key)
+
+    # New method to generate ForeignKeyConstraint aspects
+    def gen_foreign_keys(
+        self,
+        table: BigqueryTable,
+        dataset_name: str,
+        project_id: str,
+    ) -> Iterable[ForeignKeyConstraint]:
+        table_id = f"{project_id}.{dataset_name}.{table.name}"
+        foreign_keys: List[BigqueryTableConstraint] = list(
+            filter(lambda x: x.type == "FOREIGN KEY", table.constraints)
+        )
+        for key, group in groupby(
+            foreign_keys,
+            lambda x: f"{x.referenced_project_id}.{x.referenced_dataset}.{x.referenced_table_name}",
+        ):
+            dataset_urn = make_dataset_urn_with_platform_instance(
+                platform="bigquery",
+                name=table_id,
+                platform_instance=self.config.platform_instance,
+                env=self.config.env,
+            )
+
+            foreign_dataset = make_dataset_urn_with_platform_instance(
+                platform="bigquery",
+                name=key,
+                platform_instance=self.config.platform_instance,
+                env=self.config.env,
+            )
+            source_fields: List[str] = list()
+            referenced_fields: List[str] = list()
+
+            for item in group:
+                source_field = make_schema_field_urn(
+                    parent_urn=dataset_urn, field_path=item.field_path
+                )
+                assert item.referenced_column_name
+                referenced_field = make_schema_field_urn(
+                    parent_urn=foreign_dataset, field_path=item.referenced_column_name
+                )
+
+                source_fields.append(source_field)
+                referenced_fields.append(referenced_field)
+
+            foreign_key_aspect = ForeignKeyConstraint(
+                name=key,
+                foreignFields=referenced_fields,
+                sourceFields=source_fields,
+                foreignDataset=foreign_dataset,
+            )
+
+            yield foreign_key_aspect
 
     def gen_table_dataset_workunits(
         self,
@@ -724,13 +818,26 @@ def gen_table_dataset_workunits(
         tags_to_add = None
         if table.labels and self.config.capture_table_label_as_tag:
             tags_to_add = []
-            tags_to_add.extend(
-                [
-                    self.make_tag_from_label(k, v)
-                    for k, v in table.labels.items()
-                    if is_tag_allowed(self.config.capture_table_label_as_tag, k)
-                ]
-            )
+            for k, v in table.labels.items():
+                if is_tag_allowed(self.config.capture_table_label_as_tag, k):
+                    tag_urn = TagUrn.from_string(self.make_tag_urn_from_label(k, v))
+                    try:
+                        label = BigQueryLabel(key=k, value=v)
+                        platform_resource: PlatformResource = self.platform_resource_helper.generate_label_platform_resource(
+                            label, tag_urn, managed_by_datahub=False
+                        )
+                        label_info: BigQueryLabelInfo = platform_resource.resource_info.value.as_pydantic_object(  # type: ignore
+                            BigQueryLabelInfo
+                        )
+                        tag_urn = TagUrn.from_string(label_info.datahub_urn)
+
+                        for mcpw in platform_resource.to_mcps():
+                            yield mcpw.as_workunit()
+                    except ValueError as e:
+                        logger.warning(
+                            f"Failed to generate platform resource for label {k}:{v}: {e}"
+                        )
+                    tags_to_add.append(tag_urn.urn())
 
         yield from self.gen_dataset_workunits(
             table=table,
@@ -749,13 +856,29 @@ def gen_view_dataset_workunits(
         project_id: str,
         dataset_name: str,
     ) -> Iterable[MetadataWorkUnit]:
-        tags_to_add = None
+        tags_to_add = []
         if table.labels and self.config.capture_view_label_as_tag:
-            tags_to_add = [
-                self.make_tag_from_label(k, v)
-                for k, v in table.labels.items()
-                if is_tag_allowed(self.config.capture_view_label_as_tag, k)
-            ]
+            for k, v in table.labels.items():
+                if is_tag_allowed(self.config.capture_view_label_as_tag, k):
+                    tag_urn = TagUrn.from_string(self.make_tag_urn_from_label(k, v))
+                    try:
+                        label = BigQueryLabel(key=k, value=v)
+                        platform_resource: PlatformResource = self.platform_resource_helper.generate_label_platform_resource(
+                            label, tag_urn, managed_by_datahub=False
+                        )
+                        label_info: BigQueryLabelInfo = platform_resource.resource_info.value.as_pydantic_object(  # type: ignore
+                            BigQueryLabelInfo
+                        )
+                        tag_urn = TagUrn.from_string(label_info.datahub_urn)
+
+                        for mcpw in platform_resource.to_mcps():
+                            yield mcpw.as_workunit()
+                    except ValueError as e:
+                        logger.warning(
+                            f"Failed to generate platform resource for label {k}:{v}: {e}"
+                        )
+
+                    tags_to_add.append(tag_urn.urn())
         yield from self.gen_dataset_workunits(
             table=table,
             columns=columns,
@@ -818,6 +941,15 @@ def gen_dataset_workunits(
             project_id, dataset_name, table.name
         )
 
+        # Added for bigquery to gcs lineage extraction
+        if (
+            isinstance(table, BigqueryTable)
+            and table.table_type == "EXTERNAL"
+            and table.ddl is not None
+            and re.search(self.bq_external_table_pattern, table.ddl, re.IGNORECASE)
+        ):
+            self.external_tables[dataset_urn] = table
+
         status = Status(removed=False)
         yield MetadataChangeProposalWrapper(
             entityUrn=dataset_urn, aspect=status
@@ -898,7 +1030,17 @@ def gen_tags_aspect_workunit(
             entityUrn=dataset_urn, aspect=tags
         ).as_workunit()
 
-    def gen_schema_fields(self, columns: List[BigqueryColumn]) -> List[SchemaField]:
+    def is_primary_key(
+        self, field_path: str, constraints: List[BigqueryTableConstraint]
+    ) -> bool:
+        for constraint in constraints:
+            if constraint.field_path == field_path and constraint.type == "PRIMARY KEY":
+                return True
+        return False
+
+    def gen_schema_fields(
+        self, columns: List[BigqueryColumn], constraints: List[BigqueryTableConstraint]
+    ) -> List[SchemaField]:
         schema_fields: List[SchemaField] = []
 
         # Below line affects HiveColumnToAvroConverter._STRUCT_TYPE_SEPARATOR in global scope
@@ -959,6 +1101,7 @@ def gen_schema_fields(self, columns: List[BigqueryColumn]) -> List[SchemaField]:
                         self.BIGQUERY_FIELD_TYPE_MAPPINGS.get(col.data_type, NullType)()
                     ),
                     isPartitioningKey=col.is_partition_column,
+                    isPartOfKey=self.is_primary_key(col.field_path, constraints),
                     nativeDataType=col.data_type,
                     description=col.comment,
                     nullable=col.is_nullable,
@@ -978,6 +1121,15 @@ def gen_schema_metadata(
         columns: List[BigqueryColumn],
         dataset_name: BigqueryTableIdentifier,
     ) -> MetadataWorkUnit:
+        foreign_keys: List[ForeignKeyConstraint] = []
+        # Foreign keys only make sense for tables
+        if isinstance(table, BigqueryTable):
+            foreign_keys = list(
+                self.gen_foreign_keys(
+                    table, dataset_name.dataset, dataset_name.project_id
+                )
+            )
+
         schema_metadata = SchemaMetadata(
             schemaName=str(dataset_name),
             platform=self.identifiers.make_data_platform_urn(),
@@ -985,7 +1137,15 @@ def gen_schema_metadata(
             hash="",
             platformSchema=MySqlDDL(tableSchema=""),
             # fields=[],
-            fields=self.gen_schema_fields(columns),
+            fields=self.gen_schema_fields(
+                columns,
+                (
+                    table.constraints
+                    if (isinstance(table, BigqueryTable) and table.constraints)
+                    else []
+                ),
+            ),
+            foreignKeys=foreign_keys if foreign_keys else None,
         )
 
         if self.config.lineage_parse_view_ddl or self.config.lineage_use_sql_parser:
@@ -1004,14 +1164,9 @@ def get_tables_for_dataset(
     ) -> Iterable[BigqueryTable]:
         # In bigquery there is no way to query all tables in a Project id
         with PerfTimer() as timer:
-
-            # PARTITIONS INFORMATION_SCHEMA view is not available for BigLake tables
-            # based on Amazon S3 and Blob Storage data.
-            # https://cloud.google.com/bigquery/docs/omni-introduction#limitations
-            # Omni Locations - https://cloud.google.com/bigquery/docs/omni-introduction#locations
-            with_partitions = self.config.have_table_data_read_permission and not (
-                dataset.location
-                and dataset.location.lower().startswith(("aws-", "azure-"))
+            with_partitions = (
+                self.config.have_table_data_read_permission
+                and dataset.supports_table_partitions()
             )
 
             # Partitions view throw exception if we try to query partition info for too many tables
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
index c9d0738bea7dc..321b1b6207fab 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
@@ -1,6 +1,8 @@
 import collections
 import itertools
+import json
 import logging
+import re
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import (
@@ -15,17 +17,20 @@
     Tuple,
     Union,
 )
+from urllib.parse import urlparse
 
 import humanfriendly
 import sqlglot
 from google.cloud.datacatalog import lineage_v1
 from google.cloud.logging_v2.client import Client as GCPLoggingClient
 
+from datahub.api.entities.dataset.dataset import Dataset
 from datahub.configuration.pattern_utils import is_schema_allowed
 from datahub.emitter import mce_builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.source_helpers import auto_workunit
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import (
     AuditLogEntry,
     BigQueryAuditMetadata,
@@ -51,16 +56,19 @@
     BQ_FILTER_RULE_TEMPLATE_V2_LINEAGE,
     bigquery_audit_metadata_query_template_lineage,
 )
+from datahub.ingestion.source.gcs import gcs_utils
 from datahub.ingestion.source.state.redundant_run_skip_handler import (
     RedundantLineageRunSkipHandler,
 )
 from datahub.ingestion.source_report.ingestion_stage import LINEAGE_EXTRACTION
+from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaMetadata
 from datahub.metadata.schema_classes import (
     AuditStampClass,
     DatasetLineageTypeClass,
     FineGrainedLineageClass,
     FineGrainedLineageDownstreamTypeClass,
     FineGrainedLineageUpstreamTypeClass,
+    SchemaMetadataClass,
     UpstreamClass,
     UpstreamLineageClass,
 )
@@ -247,6 +255,7 @@ def __init__(
             format_queries=True,
         )
         self.report.sql_aggregator = self.aggregator.report
+        self.gcs_uris_regex = re.compile(r"uris=\[([^\]]+)\]")
 
     def get_time_window(self) -> Tuple[datetime, datetime]:
         if self.redundant_run_skip_handler:
@@ -918,3 +927,188 @@ def test_capability(self, project_id: str) -> None:
     def report_status(self, step: str, status: bool) -> None:
         if self.redundant_run_skip_handler:
             self.redundant_run_skip_handler.report_current_run_status(step, status)
+
+    def gen_lineage_workunits_for_external_table(
+        self,
+        dataset_urn: str,
+        ddl: Optional[str],
+        graph: Optional[DataHubGraph] = None,
+    ) -> Iterable[MetadataWorkUnit]:
+        if not ddl:
+            return
+
+        # Expect URIs in `uris=[""]` format
+        uris_match = self.gcs_uris_regex.search(ddl)
+        if not uris_match:
+            self.report.num_skipped_external_table_lineage += 1
+            logger.warning(f"Unable to parse GCS URI from the provided DDL {ddl}.")
+            return
+
+        uris_str = uris_match.group(1)
+        try:
+            source_uris = json.loads(f"[{uris_str}]")
+        except json.JSONDecodeError as e:
+            self.report.num_skipped_external_table_lineage += 1
+            logger.warning(
+                f"Json load failed on loading source uri with error: {e}. The field value was: {uris_str}"
+            )
+            return
+
+        lineage_info = self.get_lineage_for_external_table(
+            dataset_urn=dataset_urn,
+            source_uris=source_uris,
+            graph=graph,
+        )
+
+        if lineage_info:
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn, aspect=lineage_info
+            ).as_workunit()
+
+    def get_lineage_for_external_table(
+        self,
+        dataset_urn: str,
+        source_uris: List[str],
+        graph: Optional[DataHubGraph] = None,
+    ) -> Optional[UpstreamLineageClass]:
+        upstreams_list: List[UpstreamClass] = []
+        fine_grained_lineages: List[FineGrainedLineageClass] = []
+        gcs_urns: Set[str] = set()
+
+        for source_uri in source_uris:
+            # Check that storage_location have the gs:// prefix.
+            # Right now we are only supporting GCS lineage
+            if not gcs_utils.is_gcs_uri(source_uri):
+                continue
+            gcs_path = self._get_gcs_path(source_uri)
+
+            if gcs_path is None:
+                continue
+
+            path = gcs_utils.strip_gcs_prefix(gcs_path)
+            urn = mce_builder.make_dataset_urn_with_platform_instance(
+                platform="gcs",
+                name=path,
+                env=self.config.env,
+                platform_instance=(
+                    self.config.platform_instance
+                    if self.config.platform_instance is not None
+                    else None
+                ),
+            )
+            gcs_urns.add(urn)
+
+        upstreams_list.extend(
+            [
+                UpstreamClass(
+                    dataset=source_dataset_urn,
+                    type=DatasetLineageTypeClass.COPY,
+                )
+                for source_dataset_urn in gcs_urns
+            ]
+        )
+
+        if not upstreams_list:
+            return None
+
+        if self.config.include_column_lineage_with_gcs:
+            assert graph
+            schema_metadata: Optional[SchemaMetadataClass] = graph.get_schema_metadata(
+                dataset_urn
+            )
+            for gcs_dataset_urn in gcs_urns:
+                schema_metadata_for_gcs: Optional[
+                    SchemaMetadataClass
+                ] = graph.get_schema_metadata(gcs_dataset_urn)
+                if schema_metadata and schema_metadata_for_gcs:
+                    fine_grained_lineage = self.get_fine_grained_lineages_with_gcs(
+                        dataset_urn,
+                        gcs_dataset_urn,
+                        schema_metadata,
+                        schema_metadata_for_gcs,
+                    )
+                    if not fine_grained_lineage:
+                        logger.warning(
+                            f"Failed to retrieve fine-grained lineage for dataset {dataset_urn} and GCS {gcs_dataset_urn}. "
+                            f"Check schema metadata: {schema_metadata} and GCS metadata: {schema_metadata_for_gcs}."
+                        )
+                        continue
+
+                    fine_grained_lineages.extend(fine_grained_lineage)
+
+        upstream_lineage = UpstreamLineageClass(
+            upstreams=upstreams_list, fineGrainedLineages=fine_grained_lineages or None
+        )
+        return upstream_lineage
+
+    def _get_gcs_path(self, path: str) -> Optional[str]:
+        if self.config.gcs_lineage_config:
+            for path_spec in self.config.gcs_lineage_config.path_specs:
+                if not path_spec.allowed(path):
+                    logger.debug(
+                        f"Skipping gcs path {path} as it does not match any path spec."
+                    )
+                    self.report.num_lineage_dropped_gcs_path += 1
+                    continue
+
+                _, table_path = path_spec.extract_table_name_and_path(path)
+                return table_path
+
+            if (
+                self.config.gcs_lineage_config.ignore_non_path_spec_path
+                and len(self.config.gcs_lineage_config.path_specs) > 0
+            ):
+                self.report.num_lineage_dropped_gcs_path += 1
+                logger.debug(
+                    f"Skipping gcs path {path} as it does not match any path spec."
+                )
+                return None
+
+            if self.config.gcs_lineage_config.strip_urls:
+                if "/" in urlparse(path).path:
+                    return str(path.rsplit("/", 1)[0])
+
+        return path
+
+    def get_fine_grained_lineages_with_gcs(
+        self,
+        dataset_urn: str,
+        gcs_dataset_urn: str,
+        schema_metadata: SchemaMetadata,
+        schema_metadata_for_gcs: SchemaMetadata,
+    ) -> Optional[List[FineGrainedLineageClass]]:
+        def simplify_field_path(field_path):
+            return Dataset._simplify_field_path(field_path)
+
+        if schema_metadata and schema_metadata_for_gcs:
+            fine_grained_lineages: List[FineGrainedLineageClass] = []
+            for field in schema_metadata.fields:
+                field_path_v1 = simplify_field_path(field.fieldPath)
+                matching_gcs_field = next(
+                    (
+                        f
+                        for f in schema_metadata_for_gcs.fields
+                        if simplify_field_path(f.fieldPath) == field_path_v1
+                    ),
+                    None,
+                )
+                if matching_gcs_field:
+                    fine_grained_lineages.append(
+                        FineGrainedLineageClass(
+                            downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD,
+                            downstreams=[
+                                mce_builder.make_schema_field_urn(
+                                    dataset_urn, field_path_v1
+                                )
+                            ],
+                            upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+                            upstreams=[
+                                mce_builder.make_schema_field_urn(
+                                    gcs_dataset_urn,
+                                    simplify_field_path(matching_gcs_field.fieldPath),
+                                )
+                            ],
+                        )
+                    )
+            return fine_grained_lineages
+        return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
index 6af8166fbf70c..182ae2265cb16 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
@@ -166,12 +166,6 @@ def get_workunits(
                 normalized_table_name = BigqueryTableIdentifier(
                     project_id=project_id, dataset=dataset, table=table.name
                 ).get_table_name()
-                for column in table.columns_ignore_from_profiling:
-                    # Profiler has issues with complex types (array, struct, geography, json), so we deny those types from profiling
-                    # We also filter columns without data type as it means that column is part of a complex type.
-                    self.config.profile_pattern.deny.append(
-                        f"^{normalized_table_name}.{column}$"
-                    )
 
                 if table.external and not self.config.profiling.profile_external_tables:
                     self.report.profiling_skipped_other[f"{project_id}.{dataset}"] += 1
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py
index 6155619d144c4..2ac40a48de4cc 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py
@@ -294,6 +294,30 @@ class BigqueryQuery:
 ORDER BY
   table_catalog, table_schema, table_name, ordinal_position ASC, data_type DESC"""
 
+    constraints_for_table: str = """
+select
+    kcu.constraint_name as constraint_name,
+    kcu.table_catalog as table_catalog,
+    kcu.table_schema as table_schema,
+    kcu.table_name as table_name,
+    kcu.column_name as column_name,
+    tc.constraint_type,
+    ccu.table_catalog as referenced_catalog,
+    ccu.table_schema as referenced_schema,
+    ccu.table_name as referenced_table,
+    ccu.column_name as referenced_column
+from
+     `{project_id}`.`{dataset_name}`.INFORMATION_SCHEMA.KEY_COLUMN_USAGE as kcu
+join  `{project_id}`.`{dataset_name}`.INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE as ccu on
+    kcu.constraint_catalog = ccu.constraint_catalog
+    and kcu.constraint_schema = ccu.constraint_schema
+    and kcu.constraint_name = ccu.constraint_name
+join `{project_id}`.`{dataset_name}`.INFORMATION_SCHEMA.TABLE_CONSTRAINTS as tc on
+    tc.constraint_catalog = ccu.constraint_catalog
+    and tc.constraint_schema = ccu.constraint_schema
+    and tc.constraint_name = ccu.constraint_name
+"""
+
 
 BQ_FILTER_RULE_TEMPLATE_V2_LINEAGE = """
 resource.type=("bigquery_project")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py
index b4a443673b9a9..08c9beaa73c53 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py
@@ -2,7 +2,7 @@
 import logging
 import pathlib
 import tempfile
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Collection, Dict, Iterable, List, Optional, TypedDict
 
 from google.cloud.bigquery import Client
@@ -13,6 +13,7 @@
     BaseTimeWindowConfig,
     get_time_bucket,
 )
+from datahub.ingestion.api.closeable import Closeable
 from datahub.ingestion.api.source import SourceReport
 from datahub.ingestion.api.source_helpers import auto_workunit
 from datahub.ingestion.api.workunit import MetadataWorkUnit
@@ -48,6 +49,7 @@
     FileBackedDict,
     FileBackedList,
 )
+from datahub.utilities.progress_timer import ProgressTimer
 from datahub.utilities.time import datetime_to_ts_millis
 
 logger = logging.getLogger(__name__)
@@ -114,7 +116,7 @@ class BigQueryQueriesExtractorConfig(BigQueryBaseConfig):
     )
 
 
-class BigQueryQueriesExtractor:
+class BigQueryQueriesExtractor(Closeable):
     """
     Extracts query audit log and generates usage/lineage/operation workunits.
 
@@ -181,6 +183,7 @@ def __init__(
             is_allowed_table=self.is_allowed_table,
             format_queries=False,
         )
+
         self.report.sql_aggregator = self.aggregator.report
         self.report.num_discovered_tables = (
             len(self.discovered_tables) if self.discovered_tables else None
@@ -268,29 +271,38 @@ def get_workunits_internal(
             # Preprocessing stage that deduplicates the queries using query hash per usage bucket
             # Note: FileBackedDict is an ordered dictionary, so the order of execution of
             # queries is inherently maintained
-            queries_deduped: FileBackedDict[Dict[int, ObservedQuery]]
-            queries_deduped = self.deduplicate_queries(queries)
+            queries_deduped: FileBackedDict[
+                Dict[int, ObservedQuery]
+            ] = self.deduplicate_queries(queries)
             self.report.num_unique_queries = len(queries_deduped)
             logger.info(f"Found {self.report.num_unique_queries} unique queries")
 
-        with self.report.audit_log_load_timer:
-            i = 0
-            for _, query_instances in queries_deduped.items():
+        with self.report.audit_log_load_timer, queries_deduped:
+            log_timer = ProgressTimer(timedelta(minutes=1))
+            report_timer = ProgressTimer(timedelta(minutes=5))
+
+            for i, (_, query_instances) in enumerate(queries_deduped.items()):
                 for query in query_instances.values():
-                    if i > 0 and i % 10000 == 0:
-                        logger.info(f"Added {i} query log entries to SQL aggregator")
-                        if self.report.sql_aggregator:
-                            logger.info(self.report.sql_aggregator.as_string())
+                    if log_timer.should_report():
+                        logger.info(
+                            f"Added {i} deduplicated query log entries to SQL aggregator"
+                        )
+
+                    if report_timer.should_report() and self.report.sql_aggregator:
+                        logger.info(self.report.sql_aggregator.as_string())
 
                     self.aggregator.add(query)
-                    i += 1
 
         yield from auto_workunit(self.aggregator.gen_metadata())
 
+        if not use_cached_audit_log:
+            queries.close()
+            shared_connection.close()
+            audit_log_file.unlink(missing_ok=True)
+
     def deduplicate_queries(
         self, queries: FileBackedList[ObservedQuery]
     ) -> FileBackedDict[Dict[int, ObservedQuery]]:
-
         # This fingerprint based deduplication is done here to reduce performance hit due to
         # repetitive sql parsing while adding observed query to aggregator that would otherwise
         # parse same query multiple times. In future, aggregator may absorb this deduplication.
@@ -328,7 +340,6 @@ def deduplicate_queries(
         return queries_deduped
 
     def fetch_query_log(self, project: BigqueryProject) -> Iterable[ObservedQuery]:
-
         # Multi-regions from https://cloud.google.com/bigquery/docs/locations#supported_locations
         regions = self.config.region_qualifiers
 
@@ -341,7 +352,6 @@ def fetch_query_log(self, project: BigqueryProject) -> Iterable[ObservedQuery]:
     def fetch_region_query_log(
         self, project: BigqueryProject, region: str
     ) -> Iterable[ObservedQuery]:
-
         # Each region needs to be a different query
         query_log_query = _build_enriched_query_log_query(
             project_id=project.id,
@@ -404,6 +414,9 @@ def _parse_audit_log_row(self, row: BigQueryJob) -> ObservedQuery:
 
         return entry
 
+    def close(self) -> None:
+        self.aggregator.close()
+
 
 def _extract_query_text(row: BigQueryJob) -> str:
     # We wrap select statements in a CTE to make them parseable as DML statement.
@@ -435,7 +448,6 @@ def _build_enriched_query_log_query(
     start_time: datetime,
     end_time: datetime,
 ) -> str:
-
     audit_start_time = start_time.strftime(BQ_DATETIME_FORMAT)
     audit_end_time = end_time.strftime(BQ_DATETIME_FORMAT)
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py
new file mode 100644
index 0000000000000..dcdccc08ce048
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py
@@ -0,0 +1,475 @@
+import dataclasses
+import json
+import logging
+from typing import Any, Dict, Iterable, List, Optional
+
+from datahub.emitter.mce_builder import (
+    make_data_platform_urn,
+    make_dataplatform_instance_urn,
+    make_dataset_urn_with_platform_instance,
+    make_schema_field_urn,
+)
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.mcp_builder import (
+    ContainerKey,
+    add_dataset_to_container,
+    gen_containers,
+)
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SourceCapability,
+    SupportStatus,
+    capability,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import MetadataWorkUnitProcessor
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.cassandra.cassandra_api import (
+    CassandraAPI,
+    CassandraColumn,
+    CassandraEntities,
+    CassandraKeyspace,
+    CassandraTable,
+    CassandraView,
+)
+from datahub.ingestion.source.cassandra.cassandra_config import CassandraSourceConfig
+from datahub.ingestion.source.cassandra.cassandra_profiling import CassandraProfiler
+from datahub.ingestion.source.cassandra.cassandra_utils import (
+    SYSTEM_KEYSPACE_LIST,
+    CassandraSourceReport,
+    CassandraToSchemaFieldConverter,
+)
+from datahub.ingestion.source.common.subtypes import (
+    DatasetContainerSubTypes,
+    DatasetSubTypes,
+)
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionSourceBase,
+)
+from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass
+from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+    SchemaField,
+    SchemaMetadata,
+)
+from datahub.metadata.schema_classes import (
+    DataPlatformInstanceClass,
+    DatasetLineageTypeClass,
+    DatasetPropertiesClass,
+    FineGrainedLineageClass,
+    FineGrainedLineageDownstreamTypeClass,
+    FineGrainedLineageUpstreamTypeClass,
+    OtherSchemaClass,
+    SubTypesClass,
+    UpstreamClass,
+    UpstreamLineageClass,
+    ViewPropertiesClass,
+)
+
+logger = logging.getLogger(__name__)
+
+PLATFORM_NAME_IN_DATAHUB = "cassandra"
+
+
+class KeyspaceKey(ContainerKey):
+    keyspace: str
+
+
+@platform_name("Cassandra")
+@config_class(CassandraSourceConfig)
+@support_status(SupportStatus.INCUBATING)
+@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
+@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(
+    SourceCapability.DELETION_DETECTION,
+    "Optionally enabled via `stateful_ingestion.remove_stale_metadata`",
+    supported=True,
+)
+class CassandraSource(StatefulIngestionSourceBase):
+
+    """
+    This plugin extracts the following:
+
+    - Metadata for tables
+    - Column types associated with each table column
+    - The keyspace each table belongs to
+    """
+
+    config: CassandraSourceConfig
+    report: CassandraSourceReport
+    platform: str
+
+    def __init__(self, ctx: PipelineContext, config: CassandraSourceConfig):
+        super().__init__(config, ctx)
+        self.ctx = ctx
+        self.platform = PLATFORM_NAME_IN_DATAHUB
+        self.config = config
+        self.report = CassandraSourceReport()
+        self.cassandra_api = CassandraAPI(config, self.report)
+        self.cassandra_data = CassandraEntities()
+        # For profiling
+        self.profiler = CassandraProfiler(config, self.report, self.cassandra_api)
+
+    @classmethod
+    def create(cls, config_dict, ctx):
+        config = CassandraSourceConfig.parse_obj(config_dict)
+        return cls(ctx, config)
+
+    def get_platform(self) -> str:
+        return PLATFORM_NAME_IN_DATAHUB
+
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [
+            *super().get_workunit_processors(),
+            StaleEntityRemovalHandler.create(
+                self, self.config, self.ctx
+            ).workunit_processor,
+        ]
+
+    def get_workunits_internal(
+        self,
+    ) -> Iterable[MetadataWorkUnit]:
+        if not self.cassandra_api.authenticate():
+            return
+        keyspaces: List[CassandraKeyspace] = self.cassandra_api.get_keyspaces()
+        for keyspace in keyspaces:
+            keyspace_name: str = keyspace.keyspace_name
+            if keyspace_name in SYSTEM_KEYSPACE_LIST:
+                continue
+
+            if not self.config.keyspace_pattern.allowed(keyspace_name):
+                self.report.report_dropped(keyspace_name)
+                continue
+
+            yield from self._generate_keyspace_container(keyspace)
+
+            try:
+                yield from self._extract_tables_from_keyspace(keyspace_name)
+            except Exception as e:
+                self.report.num_tables_failed += 1
+                self.report.failure(
+                    message="Failed to extract table metadata for keyspace",
+                    context=keyspace_name,
+                    exc=e,
+                )
+            try:
+                yield from self._extract_views_from_keyspace(keyspace_name)
+            except Exception as e:
+                self.report.num_views_failed += 1
+                self.report.failure(
+                    message="Failed to extract view metadata for keyspace ",
+                    context=keyspace_name,
+                    exc=e,
+                )
+
+        # Profiling
+        if self.config.is_profiling_enabled():
+            yield from self.profiler.get_workunits(self.cassandra_data)
+
+    def _generate_keyspace_container(
+        self, keyspace: CassandraKeyspace
+    ) -> Iterable[MetadataWorkUnit]:
+        keyspace_container_key = self._generate_keyspace_container_key(
+            keyspace.keyspace_name
+        )
+        yield from gen_containers(
+            container_key=keyspace_container_key,
+            name=keyspace.keyspace_name,
+            qualified_name=keyspace.keyspace_name,
+            extra_properties={
+                "durable_writes": str(keyspace.durable_writes),
+                "replication": json.dumps(keyspace.replication),
+            },
+            sub_types=[DatasetContainerSubTypes.KEYSPACE],
+        )
+
+    def _generate_keyspace_container_key(self, keyspace_name: str) -> ContainerKey:
+        return KeyspaceKey(
+            keyspace=keyspace_name,
+            platform=self.platform,
+            instance=self.config.platform_instance,
+            env=self.config.env,
+        )
+
+    # get all tables for a given keyspace, iterate over them to extract column metadata
+    def _extract_tables_from_keyspace(
+        self, keyspace_name: str
+    ) -> Iterable[MetadataWorkUnit]:
+        self.cassandra_data.keyspaces.append(keyspace_name)
+        tables: List[CassandraTable] = self.cassandra_api.get_tables(keyspace_name)
+        for table in tables:
+            # define the dataset urn for this table to be used downstream
+            table_name: str = table.table_name
+            dataset_name: str = f"{keyspace_name}.{table_name}"
+
+            if not self.config.table_pattern.allowed(dataset_name):
+                self.report.report_dropped(dataset_name)
+                continue
+
+            self.cassandra_data.tables.setdefault(keyspace_name, []).append(table_name)
+            self.report.report_entity_scanned(dataset_name, ent_type="Table")
+
+            dataset_urn = make_dataset_urn_with_platform_instance(
+                platform=self.platform,
+                name=dataset_name,
+                env=self.config.env,
+                platform_instance=self.config.platform_instance,
+            )
+
+            # 1. Extract columns from table, then construct and emit the schemaMetadata aspect.
+            try:
+                yield from self._extract_columns_from_table(
+                    keyspace_name, table_name, dataset_urn
+                )
+            except Exception as e:
+                self.report.failure(
+                    message="Failed to extract columns from table",
+                    context=table_name,
+                    exc=e,
+                )
+
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=StatusClass(removed=False),
+            ).as_workunit()
+
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=SubTypesClass(
+                    typeNames=[
+                        DatasetSubTypes.TABLE,
+                    ]
+                ),
+            ).as_workunit()
+
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=DatasetPropertiesClass(
+                    name=table_name,
+                    qualifiedName=f"{keyspace_name}.{table_name}",
+                    description=table.comment,
+                    customProperties={
+                        "bloom_filter_fp_chance": str(table.bloom_filter_fp_chance),
+                        "caching": json.dumps(table.caching),
+                        "compaction": json.dumps(table.compaction),
+                        "compression": json.dumps(table.compression),
+                        "crc_check_chance": str(table.crc_check_chance),
+                        "dclocal_read_repair_chance": str(
+                            table.dclocal_read_repair_chance
+                        ),
+                        "default_time_to_live": str(table.default_time_to_live),
+                        "extensions": json.dumps(table.extensions),
+                        "gc_grace_seconds": str(table.gc_grace_seconds),
+                        "max_index_interval": str(table.max_index_interval),
+                        "min_index_interval": str(table.min_index_interval),
+                        "memtable_flush_period_in_ms": str(
+                            table.memtable_flush_period_in_ms
+                        ),
+                        "read_repair_chance": str(table.read_repair_chance),
+                        "speculative_retry": str(table.speculative_retry),
+                    },
+                ),
+            ).as_workunit()
+
+            yield from add_dataset_to_container(
+                container_key=self._generate_keyspace_container_key(keyspace_name),
+                dataset_urn=dataset_urn,
+            )
+
+            if self.config.platform_instance:
+                yield MetadataChangeProposalWrapper(
+                    entityUrn=dataset_urn,
+                    aspect=DataPlatformInstanceClass(
+                        platform=make_data_platform_urn(self.platform),
+                        instance=make_dataplatform_instance_urn(
+                            self.platform, self.config.platform_instance
+                        ),
+                    ),
+                ).as_workunit()
+
+    # get all columns for a given table, iterate over them to extract column metadata
+    def _extract_columns_from_table(
+        self, keyspace_name: str, table_name: str, dataset_urn: str
+    ) -> Iterable[MetadataWorkUnit]:
+        column_infos: List[CassandraColumn] = self.cassandra_api.get_columns(
+            keyspace_name, table_name
+        )
+        schema_fields: List[SchemaField] = list(
+            CassandraToSchemaFieldConverter.get_schema_fields(column_infos)
+        )
+        if not schema_fields:
+            self.report.report_warning(
+                message="Table has no columns, skipping", context=table_name
+            )
+            return
+
+        jsonable_column_infos: List[Dict[str, Any]] = []
+        for column in column_infos:
+            self.cassandra_data.columns.setdefault(table_name, []).append(column)
+            jsonable_column_infos.append(dataclasses.asdict(column))
+
+        schema_metadata: SchemaMetadata = SchemaMetadata(
+            schemaName=table_name,
+            platform=make_data_platform_urn(self.platform),
+            version=0,
+            hash="",
+            platformSchema=OtherSchemaClass(
+                rawSchema=json.dumps(jsonable_column_infos)
+            ),
+            fields=schema_fields,
+        )
+
+        yield MetadataChangeProposalWrapper(
+            entityUrn=dataset_urn,
+            aspect=schema_metadata,
+        ).as_workunit()
+
+    def _extract_views_from_keyspace(
+        self, keyspace_name: str
+    ) -> Iterable[MetadataWorkUnit]:
+        views: List[CassandraView] = self.cassandra_api.get_views(keyspace_name)
+        for view in views:
+            view_name: str = view.view_name
+            dataset_name: str = f"{keyspace_name}.{view_name}"
+            self.report.report_entity_scanned(dataset_name)
+            dataset_urn: str = make_dataset_urn_with_platform_instance(
+                platform=self.platform,
+                name=dataset_name,
+                env=self.config.env,
+                platform_instance=self.config.platform_instance,
+            )
+
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=StatusClass(removed=False),
+            ).as_workunit()
+
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=SubTypesClass(
+                    typeNames=[
+                        DatasetSubTypes.VIEW,
+                    ]
+                ),
+            ).as_workunit()
+
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=ViewPropertiesClass(
+                    materialized=True,
+                    viewLogic=view.where_clause,  # Use the WHERE clause as view logic
+                    viewLanguage="CQL",  # Use "CQL" as the language
+                ),
+            ).as_workunit()
+
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=DatasetPropertiesClass(
+                    name=view_name,
+                    qualifiedName=f"{keyspace_name}.{view_name}",
+                    description=view.comment,
+                    customProperties={
+                        "bloom_filter_fp_chance": str(view.bloom_filter_fp_chance),
+                        "caching": json.dumps(view.caching),
+                        "compaction": json.dumps(view.compaction),
+                        "compression": json.dumps(view.compression),
+                        "crc_check_chance": str(view.crc_check_chance),
+                        "include_all_columns": str(view.include_all_columns),
+                        "dclocal_read_repair_chance": str(
+                            view.dclocal_read_repair_chance
+                        ),
+                        "default_time_to_live": str(view.default_time_to_live),
+                        "extensions": json.dumps(view.extensions),
+                        "gc_grace_seconds": str(view.gc_grace_seconds),
+                        "max_index_interval": str(view.max_index_interval),
+                        "min_index_interval": str(view.min_index_interval),
+                        "memtable_flush_period_in_ms": str(
+                            view.memtable_flush_period_in_ms
+                        ),
+                        "read_repair_chance": str(view.read_repair_chance),
+                        "speculative_retry": str(view.speculative_retry),
+                    },
+                ),
+            ).as_workunit()
+
+            try:
+                yield from self._extract_columns_from_table(
+                    keyspace_name, view_name, dataset_urn
+                )
+            except Exception as e:
+                self.report.failure(
+                    message="Failed to extract columns from views",
+                    context=view_name,
+                    exc=e,
+                )
+
+            # Construct and emit lineage off of 'base_table_name'
+            # NOTE: we don't need to use 'base_table_id' since table is always in same keyspace, see https://docs.datastax.com/en/cql-oss/3.3/cql/cql_reference/cqlCreateMaterializedView.html#cqlCreateMaterializedView__keyspace-name
+            upstream_urn: str = make_dataset_urn_with_platform_instance(
+                platform=self.platform,
+                name=f"{keyspace_name}.{view.table_name}",
+                env=self.config.env,
+                platform_instance=self.config.platform_instance,
+            )
+            fineGrainedLineages = self.get_upstream_fields_of_field_in_datasource(
+                view_name, dataset_urn, upstream_urn
+            )
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=UpstreamLineageClass(
+                    upstreams=[
+                        UpstreamClass(
+                            dataset=upstream_urn,
+                            type=DatasetLineageTypeClass.VIEW,
+                        )
+                    ],
+                    fineGrainedLineages=fineGrainedLineages,
+                ),
+            ).as_workunit()
+
+            yield from add_dataset_to_container(
+                container_key=self._generate_keyspace_container_key(keyspace_name),
+                dataset_urn=dataset_urn,
+            )
+
+            if self.config.platform_instance:
+                yield MetadataChangeProposalWrapper(
+                    entityUrn=dataset_urn,
+                    aspect=DataPlatformInstanceClass(
+                        platform=make_data_platform_urn(self.platform),
+                        instance=make_dataplatform_instance_urn(
+                            self.platform, self.config.platform_instance
+                        ),
+                    ),
+                ).as_workunit()
+
+    def get_upstream_fields_of_field_in_datasource(
+        self, table_name: str, dataset_urn: str, upstream_urn: str
+    ) -> List[FineGrainedLineageClass]:
+        column_infos = self.cassandra_data.columns.get(table_name, [])
+        # Collect column-level lineage
+        fine_grained_lineages = []
+        for column_info in column_infos:
+            source_column = column_info.column_name
+            if source_column:
+                fine_grained_lineages.append(
+                    FineGrainedLineageClass(
+                        upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+                        downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD,
+                        downstreams=[make_schema_field_urn(dataset_urn, source_column)],
+                        upstreams=[make_schema_field_urn(upstream_urn, source_column)],
+                    )
+                )
+        return fine_grained_lineages
+
+    def get_report(self):
+        return self.report
+
+    def close(self):
+        self.cassandra_api.close()
+        super().close()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_api.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_api.py
new file mode 100644
index 0000000000000..4cf0613762aab
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_api.py
@@ -0,0 +1,325 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from cassandra import DriverException, OperationTimedOut
+from cassandra.auth import PlainTextAuthProvider
+from cassandra.cluster import (
+    EXEC_PROFILE_DEFAULT,
+    Cluster,
+    ExecutionProfile,
+    ProtocolVersion,
+    Session,
+)
+
+from datahub.ingestion.api.source import SourceReport
+from datahub.ingestion.source.cassandra.cassandra_config import CassandraSourceConfig
+
+
+@dataclass
+class CassandraKeyspace:
+    keyspace_name: str
+    durable_writes: bool
+    replication: Dict
+
+
+@dataclass
+class CassandraTable:
+    keyspace_name: str
+    table_name: str
+    bloom_filter_fp_chance: Optional[float]
+    caching: Optional[Dict[str, str]]
+    comment: Optional[str]
+    compaction: Optional[Dict[str, Any]]
+    compression: Optional[Dict[str, Any]]
+    crc_check_chance: Optional[float]
+    dclocal_read_repair_chance: Optional[float]
+    default_time_to_live: Optional[int]
+    extensions: Optional[Dict[str, Any]]
+    gc_grace_seconds: Optional[int]
+    max_index_interval: Optional[int]
+    memtable_flush_period_in_ms: Optional[int]
+    min_index_interval: Optional[int]
+    read_repair_chance: Optional[float]
+    speculative_retry: Optional[str]
+
+
+@dataclass
+class CassandraColumn:
+    keyspace_name: str
+    table_name: str
+    column_name: str
+    type: str
+    clustering_order: Optional[str]
+    kind: Optional[str]
+    position: Optional[int]
+
+
+@dataclass
+class CassandraView(CassandraTable):
+    view_name: str
+    include_all_columns: Optional[bool]
+    where_clause: str = ""
+
+
+@dataclass
+class CassandraEntities:
+    keyspaces: List[str] = field(default_factory=list)
+    tables: Dict[str, List[str]] = field(
+        default_factory=dict
+    )  # Maps keyspace -> tables
+    columns: Dict[str, List[CassandraColumn]] = field(
+        default_factory=dict
+    )  # Maps tables -> columns
+
+
+# - Referencing system_schema: https://docs.datastax.com/en/cql-oss/3.x/cql/cql_using/useQuerySystem.html#Table3.ColumnsinSystem_SchemaTables-Cassandra3.0 - #
+# this keyspace contains details about the cassandra cluster's keyspaces, tables, and columns
+
+
+class CassandraQueries:
+    # get all keyspaces
+    GET_KEYSPACES_QUERY = "SELECT * FROM system_schema.keyspaces"
+    # get all tables for a keyspace
+    GET_TABLES_QUERY = "SELECT * FROM system_schema.tables WHERE keyspace_name = %s"
+    # get all columns for a table
+    GET_COLUMNS_QUERY = "SELECT * FROM system_schema.columns WHERE keyspace_name = %s AND table_name = %s"
+    # get all views for a keyspace
+    GET_VIEWS_QUERY = "SELECT * FROM system_schema.views WHERE keyspace_name = %s"
+    # Row Count
+    ROW_COUNT = 'SELECT COUNT(*) AS row_count FROM {}."{}"'
+    # Column Count
+    COLUMN_COUNT = "SELECT COUNT(*) AS column_count FROM system_schema.columns WHERE keyspace_name = '{}' AND table_name = '{}'"
+
+
+class CassandraAPI:
+    def __init__(self, config: CassandraSourceConfig, report: SourceReport):
+        self.config = config
+        self.report = report
+        self._cassandra_session: Optional[Session] = None
+
+    def authenticate(self) -> bool:
+        """Establish a connection to Cassandra and return the session."""
+        try:
+            if self.config.cloud_config:
+                cloud_config = self.config.cloud_config
+                cluster_cloud_config = {
+                    "connect_timeout": cloud_config.connect_timeout,
+                    "use_default_tempdir": True,
+                    "secure_connect_bundle": cloud_config.secure_connect_bundle,
+                }
+                profile = ExecutionProfile(request_timeout=cloud_config.request_timeout)
+                auth_provider = PlainTextAuthProvider(
+                    "token",
+                    cloud_config.token,
+                )
+                cluster = Cluster(
+                    cloud=cluster_cloud_config,
+                    auth_provider=auth_provider,
+                    execution_profiles={EXEC_PROFILE_DEFAULT: profile},
+                    protocol_version=ProtocolVersion.V4,
+                )
+
+                self._cassandra_session = cluster.connect()
+                return True
+            if self.config.username and self.config.password:
+                auth_provider = PlainTextAuthProvider(
+                    username=self.config.username, password=self.config.password
+                )
+                cluster = Cluster(
+                    [self.config.contact_point],
+                    port=self.config.port,
+                    auth_provider=auth_provider,
+                    load_balancing_policy=None,
+                )
+            else:
+                cluster = Cluster(
+                    [self.config.contact_point],
+                    port=self.config.port,
+                    load_balancing_policy=None,
+                )
+
+            self._cassandra_session = cluster.connect()
+            return True
+        except OperationTimedOut as e:
+            self.report.failure(
+                message="Failed to Authenticate", context=f"{str(e.errors)}", exc=e
+            )
+            return False
+        except DriverException as e:
+            self.report.failure(message="Failed to Authenticate", exc=e)
+            return False
+        except Exception as e:
+            self.report.failure(message="Failed to authenticate to Cassandra", exc=e)
+            return False
+
+    def get(self, query: str, parameters: Optional[List] = []) -> List:
+        if not self._cassandra_session:
+            return []
+
+        resp = self._cassandra_session.execute(query, parameters)
+        return resp
+
+    def get_keyspaces(self) -> List[CassandraKeyspace]:
+        """Fetch all keyspaces."""
+        try:
+            keyspaces = self.get(CassandraQueries.GET_KEYSPACES_QUERY)
+            keyspace_list = [
+                CassandraKeyspace(
+                    keyspace_name=row.keyspace_name,
+                    durable_writes=row.durable_writes,
+                    replication=dict(row.replication),
+                )
+                for row in keyspaces
+            ]
+            return keyspace_list
+        except DriverException as e:
+            self.report.warning(
+                message="Failed to fetch keyspaces", context=f"{str(e)}", exc=e
+            )
+            return []
+        except Exception as e:
+            self.report.warning(message="Failed to fetch keyspaces", exc=e)
+            return []
+
+    def get_tables(self, keyspace_name: str) -> List[CassandraTable]:
+        """Fetch all tables for a given keyspace."""
+        try:
+            tables = self.get(CassandraQueries.GET_TABLES_QUERY, [keyspace_name])
+            table_list = [
+                CassandraTable(
+                    keyspace_name=row.keyspace_name,
+                    table_name=row.table_name,
+                    bloom_filter_fp_chance=row.bloom_filter_fp_chance,
+                    caching=dict(row.caching),
+                    comment=row.comment,
+                    compaction=dict(row.compaction),
+                    compression=dict(row.compression),
+                    crc_check_chance=row.crc_check_chance,
+                    dclocal_read_repair_chance=row.dclocal_read_repair_chance,
+                    default_time_to_live=row.default_time_to_live,
+                    extensions=dict(row.extensions),
+                    gc_grace_seconds=row.gc_grace_seconds,
+                    max_index_interval=row.max_index_interval,
+                    memtable_flush_period_in_ms=row.memtable_flush_period_in_ms,
+                    min_index_interval=row.min_index_interval,
+                    read_repair_chance=row.read_repair_chance,
+                    speculative_retry=row.speculative_retry,
+                )
+                for row in tables
+            ]
+            return table_list
+        except DriverException as e:
+            self.report.warning(
+                message="Failed to fetch tables for keyspace",
+                context=f"{str(e)}",
+                exc=e,
+            )
+            return []
+        except Exception as e:
+            self.report.warning(
+                message="Failed to fetch tables for keyspace",
+                context=f"{keyspace_name}",
+                exc=e,
+            )
+            return []
+
+    def get_columns(self, keyspace_name: str, table_name: str) -> List[CassandraColumn]:
+        """Fetch all columns for a given table."""
+        try:
+            column_infos = self.get(
+                CassandraQueries.GET_COLUMNS_QUERY, [keyspace_name, table_name]
+            )
+            column_list = [
+                CassandraColumn(
+                    keyspace_name=row.keyspace_name,
+                    table_name=row.table_name,
+                    column_name=row.column_name,
+                    clustering_order=row.clustering_order,
+                    kind=row.kind,
+                    position=row.position,
+                    type=row.type,
+                )
+                for row in column_infos
+            ]
+            return column_list
+        except DriverException as e:
+            self.report.warning(
+                message="Failed to fetch columns for table", context=f"{str(e)}", exc=e
+            )
+            return []
+        except Exception as e:
+            self.report.warning(
+                message="Failed to fetch columns for table",
+                context=f"{keyspace_name}.{table_name}",
+                exc=e,
+            )
+            return []
+
+    def get_views(self, keyspace_name: str) -> List[CassandraView]:
+        """Fetch all views for a given keyspace."""
+        try:
+            views = self.get(CassandraQueries.GET_VIEWS_QUERY, [keyspace_name])
+            view_list = [
+                CassandraView(
+                    table_name=row.base_table_name,
+                    keyspace_name=row.keyspace_name,
+                    view_name=row.view_name,
+                    bloom_filter_fp_chance=row.bloom_filter_fp_chance,
+                    caching=dict(row.caching),
+                    comment=row.comment,
+                    compaction=dict(row.compaction),
+                    compression=dict(row.compression),
+                    crc_check_chance=row.crc_check_chance,
+                    dclocal_read_repair_chance=row.dclocal_read_repair_chance,
+                    default_time_to_live=row.default_time_to_live,
+                    extensions=dict(row.extensions),
+                    gc_grace_seconds=row.gc_grace_seconds,
+                    include_all_columns=row.include_all_columns,
+                    max_index_interval=row.max_index_interval,
+                    memtable_flush_period_in_ms=row.memtable_flush_period_in_ms,
+                    min_index_interval=row.min_index_interval,
+                    read_repair_chance=row.read_repair_chance,
+                    speculative_retry=row.speculative_retry,
+                    where_clause=row.where_clause,
+                )
+                for row in views
+            ]
+            return view_list
+        except DriverException as e:
+            self.report.warning(
+                message="Failed to fetch views for keyspace", context=f"{str(e)}", exc=e
+            )
+            return []
+        except Exception as e:
+            self.report.warning(
+                message="Failed to fetch views for keyspace",
+                context=f"{keyspace_name}",
+                exc=e,
+            )
+            return []
+
+    def execute(self, query: str, limit: Optional[int] = None) -> List:
+        """Fetch stats for cassandra"""
+        try:
+            if not self._cassandra_session:
+                return []
+            if limit:
+                query = query + f" LIMIT {limit}"
+            result_set = self._cassandra_session.execute(query).all()
+            return result_set
+        except DriverException as e:
+            self.report.warning(
+                message="Failed to fetch stats for keyspace", context=str(e), exc=e
+            )
+            return []
+        except Exception:
+            self.report.warning(
+                message="Failed to fetch stats for keyspace",
+                context=f"{query}",
+            )
+            return []
+
+    def close(self):
+        """Close the Cassandra session."""
+        if self._cassandra_session:
+            self._cassandra_session.shutdown()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_config.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_config.py
new file mode 100644
index 0000000000000..340bdb68aa458
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_config.py
@@ -0,0 +1,111 @@
+from typing import Optional
+
+from pydantic import Field
+
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.source_common import (
+    EnvConfigMixin,
+    PlatformInstanceConfigMixin,
+)
+from datahub.ingestion.source.ge_profiling_config import GEProfilingBaseConfig
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StatefulStaleMetadataRemovalConfig,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionConfigBase,
+)
+from datahub.ingestion.source_config.operation_config import is_profiling_enabled
+
+# - Referencing https://docs.datastax.com/en/cql-oss/3.x/cql/cql_using/useQuerySystem.html#Table3.ColumnsinSystem_SchemaTables-Cassandra3.0 - #
+# this keyspace contains details about the cassandra cluster's keyspaces, tables, and columns
+SYSTEM_SCHEMA_KEYSPACE_NAME = "system_schema"
+
+# Reference:
+# https://docs.datastax.com/en/astra-db-serverless/databases/python-driver.html
+# https://docs.datastax.com/en/astra-db-serverless/databases/python-driver.html#production-configuration
+
+
+class CassandraCloudConfig(ConfigModel):
+    """
+    Configuration for connecting to DataStax Astra DB in the cloud.
+    """
+
+    token: str = Field(
+        description="The Astra DB application token used for authentication.",
+    )
+
+    secure_connect_bundle: str = Field(
+        description="File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax Astra DB.",
+    )
+
+    connect_timeout: int = Field(
+        default=600,
+        description="Timeout in seconds for establishing new connections to Cassandra.",
+    )
+
+    request_timeout: int = Field(
+        default=600, description="Timeout in seconds for individual Cassandra requests."
+    )
+
+
+class CassandraSourceConfig(
+    PlatformInstanceConfigMixin, StatefulIngestionConfigBase, EnvConfigMixin
+):
+    """
+    Configuration for connecting to a Cassandra or DataStax Astra DB source.
+    """
+
+    contact_point: str = Field(
+        default="localhost",
+        description="Domain or IP address of the Cassandra instance (excluding port).",
+    )
+
+    port: int = Field(
+        default=9042, description="Port number to connect to the Cassandra instance."
+    )
+
+    username: Optional[str] = Field(
+        default=None,
+        description=f"Username credential with read access to the {SYSTEM_SCHEMA_KEYSPACE_NAME} keyspace.",
+    )
+
+    password: Optional[str] = Field(
+        default=None,
+        description="Password credential associated with the specified username.",
+    )
+
+    cloud_config: Optional[CassandraCloudConfig] = Field(
+        default=None,
+        description="Configuration for cloud-based Cassandra, such as DataStax Astra DB.",
+    )
+
+    keyspace_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns to filter keyspaces for ingestion.",
+    )
+
+    table_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns to filter keyspaces.tables for ingestion.",
+    )
+
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field(
+        default=None,
+        description="Configuration for stateful ingestion and stale metadata removal.",
+    )
+
+    # Profiling
+    profile_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for tables to profile",
+    )
+
+    profiling: GEProfilingBaseConfig = Field(
+        default=GEProfilingBaseConfig(),
+        description="Configuration for profiling",
+    )
+
+    def is_profiling_enabled(self) -> bool:
+        return self.profiling.enabled and is_profiling_enabled(
+            self.profiling.operation_config
+        )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_profiling.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_profiling.py
new file mode 100644
index 0000000000000..d8ab62f1d6d91
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_profiling.py
@@ -0,0 +1,296 @@
+import logging
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterable, List, Optional
+
+import numpy as np
+from cassandra.util import OrderedMapSerializedKey, SortedSet
+
+from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.cassandra.cassandra_api import (
+    CassandraAPI,
+    CassandraColumn,
+    CassandraEntities,
+    CassandraQueries,
+)
+from datahub.ingestion.source.cassandra.cassandra_config import CassandraSourceConfig
+from datahub.ingestion.source.cassandra.cassandra_utils import CassandraSourceReport
+from datahub.ingestion.source_report.ingestion_stage import PROFILING
+from datahub.metadata.schema_classes import (
+    DatasetFieldProfileClass,
+    DatasetProfileClass,
+    QuantileClass,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ColumnMetric:
+    col_type: str = ""
+    values: List[Any] = field(default_factory=list)
+    null_count: int = 0
+    total_count: int = 0
+    distinct_count: Optional[int] = None
+    min: Optional[Any] = None
+    max: Optional[Any] = None
+    mean: Optional[float] = None
+    stdev: Optional[float] = None
+    median: Optional[float] = None
+    quantiles: Optional[List[float]] = None
+    sample_values: Optional[Any] = None
+
+
+@dataclass
+class ProfileData:
+    row_count: Optional[int] = None
+    column_count: Optional[int] = None
+    column_metrics: Dict[str, ColumnMetric] = field(default_factory=dict)
+
+
+class CassandraProfiler:
+    config: CassandraSourceConfig
+    report: CassandraSourceReport
+
+    def __init__(
+        self,
+        config: CassandraSourceConfig,
+        report: CassandraSourceReport,
+        api: CassandraAPI,
+    ) -> None:
+        self.api = api
+        self.config = config
+        self.report = report
+
+    def get_workunits(
+        self, cassandra_data: CassandraEntities
+    ) -> Iterable[MetadataWorkUnit]:
+        for keyspace_name in cassandra_data.keyspaces:
+            tables = cassandra_data.tables.get(keyspace_name, [])
+            self.report.set_ingestion_stage(keyspace_name, PROFILING)
+            with ThreadPoolExecutor(
+                max_workers=self.config.profiling.max_workers
+            ) as executor:
+                future_to_dataset = {
+                    executor.submit(
+                        self.generate_profile,
+                        keyspace_name,
+                        table_name,
+                        cassandra_data.columns.get(table_name, []),
+                    ): table_name
+                    for table_name in tables
+                }
+                for future in as_completed(future_to_dataset):
+                    table_name = future_to_dataset[future]
+                    try:
+                        yield from future.result()
+                    except Exception as exc:
+                        self.report.profiling_skipped_other[table_name] += 1
+                        self.report.failure(
+                            message="Failed to profile for table",
+                            context=f"{keyspace_name}.{table_name}",
+                            exc=exc,
+                        )
+
+    def generate_profile(
+        self,
+        keyspace_name: str,
+        table_name: str,
+        columns: List[CassandraColumn],
+    ) -> Iterable[MetadataWorkUnit]:
+        dataset_name: str = f"{keyspace_name}.{table_name}"
+        dataset_urn = make_dataset_urn_with_platform_instance(
+            platform="cassandra",
+            name=dataset_name,
+            env=self.config.env,
+            platform_instance=self.config.platform_instance,
+        )
+
+        if not columns:
+            self.report.warning(
+                message="Skipping profiling as no columns found for table",
+                context=f"{keyspace_name}.{table_name}",
+            )
+            self.report.profiling_skipped_other[table_name] += 1
+            return
+
+        if not self.config.profile_pattern.allowed(f"{keyspace_name}.{table_name}"):
+            self.report.profiling_skipped_table_profile_pattern[keyspace_name] += 1
+            logger.info(
+                f"Table {table_name} in {keyspace_name}, not allowed for profiling"
+            )
+            return
+
+        try:
+            profile_data = self.profile_table(keyspace_name, table_name, columns)
+        except Exception as e:
+            self.report.warning(
+                message="Profiling Failed",
+                context=f"{keyspace_name}.{table_name}",
+                exc=e,
+            )
+            return
+
+        profile_aspect = self.populate_profile_aspect(profile_data)
+
+        if profile_aspect:
+            self.report.report_entity_profiled(table_name)
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn, aspect=profile_aspect
+            )
+            yield mcp.as_workunit()
+
+    def populate_profile_aspect(self, profile_data: ProfileData) -> DatasetProfileClass:
+        field_profiles = [
+            self._create_field_profile(column_name, column_metrics)
+            for column_name, column_metrics in profile_data.column_metrics.items()
+        ]
+        return DatasetProfileClass(
+            timestampMillis=round(time.time() * 1000),
+            rowCount=profile_data.row_count,
+            columnCount=profile_data.column_count,
+            fieldProfiles=field_profiles,
+        )
+
+    def _create_field_profile(
+        self, field_name: str, field_stats: ColumnMetric
+    ) -> DatasetFieldProfileClass:
+        quantiles = field_stats.quantiles
+        return DatasetFieldProfileClass(
+            fieldPath=field_name,
+            uniqueCount=field_stats.distinct_count,
+            nullCount=field_stats.null_count,
+            min=str(field_stats.min) if field_stats.min else None,
+            max=str(field_stats.max) if field_stats.max else None,
+            mean=str(field_stats.mean) if field_stats.mean else None,
+            median=str(field_stats.median) if field_stats.median else None,
+            stdev=str(field_stats.stdev) if field_stats.stdev else None,
+            quantiles=[
+                QuantileClass(quantile=str(0.25), value=str(quantiles[0])),
+                QuantileClass(quantile=str(0.75), value=str(quantiles[1])),
+            ]
+            if quantiles
+            else None,
+            sampleValues=field_stats.sample_values
+            if field_stats.sample_values
+            else None,
+        )
+
+    def profile_table(
+        self, keyspace_name: str, table_name: str, columns: List[CassandraColumn]
+    ) -> ProfileData:
+        profile_data = ProfileData()
+
+        resp = self.api.execute(
+            CassandraQueries.ROW_COUNT.format(keyspace_name, table_name)
+        )
+        if resp:
+            profile_data.row_count = resp[0].row_count
+
+        profile_data.column_count = len(columns)
+
+        if not self.config.profiling.profile_table_level_only:
+            resp = self.api.execute(
+                f'SELECT {", ".join([col.column_name for col in columns])} FROM {keyspace_name}."{table_name}"'
+            )
+            profile_data.column_metrics = self._collect_column_data(resp, columns)
+
+        return self._parse_profile_results(profile_data)
+
+    def _parse_profile_results(self, profile_data: ProfileData) -> ProfileData:
+        for cl_name, column_metrics in profile_data.column_metrics.items():
+            if column_metrics.values:
+                try:
+                    self._compute_field_statistics(column_metrics)
+                except Exception as e:
+                    self.report.warning(
+                        message="Profiling Failed For Column Stats",
+                        context=cl_name,
+                        exc=e,
+                    )
+                    raise e
+        return profile_data
+
+    def _collect_column_data(
+        self, rows: List[Any], columns: List[CassandraColumn]
+    ) -> Dict[str, ColumnMetric]:
+        metrics = {column.column_name: ColumnMetric() for column in columns}
+
+        for row in rows:
+            for column in columns:
+                if self._is_skippable_type(column.type):
+                    continue
+
+                value: Any = getattr(row, column.column_name, None)
+                metric = metrics[column.column_name]
+                metric.col_type = column.type
+
+                metric.total_count += 1
+                if value is None:
+                    metric.null_count += 1
+                else:
+                    metric.values.extend(self._parse_value(value))
+
+        return metrics
+
+    def _is_skippable_type(self, data_type: str) -> bool:
+        return data_type.lower() in ["timeuuid", "blob", "frozen<tuple<tinyint, text>>"]
+
+    def _parse_value(self, value: Any) -> List[Any]:
+        if isinstance(value, SortedSet):
+            return list(value)
+        elif isinstance(value, OrderedMapSerializedKey):
+            return list(dict(value).values())
+        elif isinstance(value, list):
+            return value
+        return [value]
+
+    def _compute_field_statistics(self, column_metrics: ColumnMetric) -> None:
+        values = column_metrics.values
+        if not values:
+            return
+
+        # ByDefault Null count is added
+        if not self.config.profiling.include_field_null_count:
+            column_metrics.null_count = 0
+
+        if self.config.profiling.include_field_distinct_count:
+            column_metrics.distinct_count = len(set(values))
+
+        if self.config.profiling.include_field_min_value:
+            column_metrics.min = min(values)
+
+        if self.config.profiling.include_field_max_value:
+            column_metrics.max = max(values)
+
+        if values and self._is_numeric_type(column_metrics.col_type):
+            if self.config.profiling.include_field_mean_value:
+                column_metrics.mean = round(float(np.mean(values)), 2)
+            if self.config.profiling.include_field_stddev_value:
+                column_metrics.stdev = round(float(np.std(values)), 2)
+            if self.config.profiling.include_field_median_value:
+                column_metrics.median = round(float(np.median(values)), 2)
+            if self.config.profiling.include_field_quantiles:
+                column_metrics.quantiles = [
+                    float(np.percentile(values, 25)),
+                    float(np.percentile(values, 75)),
+                ]
+
+        if values and self.config.profiling.include_field_sample_values:
+            column_metrics.sample_values = [str(v) for v in values[:5]]
+
+    def _is_numeric_type(self, data_type: str) -> bool:
+        return data_type.lower() in [
+            "int",
+            "counter",
+            "bigint",
+            "float",
+            "double",
+            "decimal",
+            "smallint",
+            "tinyint",
+            "varint",
+        ]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_utils.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_utils.py
new file mode 100644
index 0000000000000..41d4ac7ced603
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_utils.py
@@ -0,0 +1,152 @@
+import logging
+from dataclasses import dataclass, field
+from typing import Dict, Generator, List, Optional, Type
+
+from datahub.ingestion.source.cassandra.cassandra_api import CassandraColumn
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalSourceReport,
+)
+from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
+from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+    SchemaField,
+    SchemaFieldDataType,
+)
+from datahub.metadata.schema_classes import (
+    ArrayTypeClass,
+    BooleanTypeClass,
+    BytesTypeClass,
+    DateTypeClass,
+    NullTypeClass,
+    NumberTypeClass,
+    RecordTypeClass,
+    StringTypeClass,
+    TimeTypeClass,
+)
+from datahub.utilities.lossy_collections import LossyList
+from datahub.utilities.stats_collections import TopKDict, int_top_k_dict
+
+logger = logging.getLogger(__name__)
+
+
+# we always skip over ingesting metadata about these keyspaces
+SYSTEM_KEYSPACE_LIST = set(
+    ["system", "system_auth", "system_schema", "system_distributed", "system_traces"]
+)
+
+
+@dataclass
+class CassandraSourceReport(StaleEntityRemovalSourceReport, IngestionStageReport):
+    num_tables_failed: int = 0
+    num_views_failed: int = 0
+    tables_scanned: int = 0
+    views_scanned: int = 0
+    entities_profiled: int = 0
+    filtered: LossyList[str] = field(default_factory=LossyList)
+
+    def report_entity_scanned(self, name: str, ent_type: str = "View") -> None:
+        """
+        Entity could be a view or a table
+        """
+        if ent_type == "Table":
+            self.tables_scanned += 1
+        elif ent_type == "View":
+            self.views_scanned += 1
+        else:
+            raise KeyError(f"Unknown entity {ent_type}.")
+
+    def set_ingestion_stage(self, keyspace: str, stage: str) -> None:
+        self.report_ingestion_stage_start(f"{keyspace}: {stage}")
+
+    # TODO Need to create seperate common config for profiling report
+    profiling_skipped_other: TopKDict[str, int] = field(default_factory=int_top_k_dict)
+    profiling_skipped_table_profile_pattern: TopKDict[str, int] = field(
+        default_factory=int_top_k_dict
+    )
+
+    def report_entity_profiled(self, name: str) -> None:
+        self.entities_profiled += 1
+
+    def report_dropped(self, ent_name: str) -> None:
+        self.filtered.append(ent_name)
+
+
+# This class helps convert cassandra column types to SchemaFieldDataType for use by the datahaub metadata schema
+class CassandraToSchemaFieldConverter:
+    # Mapping from cassandra field types to SchemaFieldDataType.
+    # https://cassandra.apache.org/doc/stable/cassandra/cql/types.html (version 4.1)
+    _field_type_to_schema_field_type: Dict[str, Type] = {
+        # Bool
+        "boolean": BooleanTypeClass,
+        # Binary
+        "blob": BytesTypeClass,
+        # Numbers
+        "bigint": NumberTypeClass,
+        "counter": NumberTypeClass,
+        "decimal": NumberTypeClass,
+        "double": NumberTypeClass,
+        "float": NumberTypeClass,
+        "int": NumberTypeClass,
+        "smallint": NumberTypeClass,
+        "tinyint": NumberTypeClass,
+        "varint": NumberTypeClass,
+        # Dates
+        "date": DateTypeClass,
+        # Times
+        "duration": TimeTypeClass,
+        "time": TimeTypeClass,
+        "timestamp": TimeTypeClass,
+        # Strings
+        "text": StringTypeClass,
+        "ascii": StringTypeClass,
+        "inet": StringTypeClass,
+        "timeuuid": StringTypeClass,
+        "uuid": StringTypeClass,
+        "varchar": StringTypeClass,
+        # Records
+        "geo_point": RecordTypeClass,
+        # Arrays
+        "histogram": ArrayTypeClass,
+    }
+
+    @staticmethod
+    def get_column_type(cassandra_column_type: str) -> SchemaFieldDataType:
+        type_class: Optional[
+            Type
+        ] = CassandraToSchemaFieldConverter._field_type_to_schema_field_type.get(
+            cassandra_column_type
+        )
+        if type_class is None:
+            logger.warning(
+                f"Cannot map {cassandra_column_type!r} to SchemaFieldDataType, using NullTypeClass."
+            )
+            type_class = NullTypeClass
+
+        return SchemaFieldDataType(type=type_class())
+
+    def _get_schema_fields(
+        self, cassandra_column_infos: List[CassandraColumn]
+    ) -> Generator[SchemaField, None, None]:
+        # append each schema field (sort so output is consistent)
+        for column_info in cassandra_column_infos:
+            column_name: str = column_info.column_name
+            cassandra_type: str = column_info.type
+
+            schema_field_data_type: SchemaFieldDataType = self.get_column_type(
+                cassandra_type
+            )
+            schema_field: SchemaField = SchemaField(
+                fieldPath=column_name,
+                nativeDataType=cassandra_type,
+                type=schema_field_data_type,
+                description=None,
+                nullable=True,
+                recursive=False,
+            )
+            yield schema_field
+
+    @classmethod
+    def get_schema_fields(
+        cls, cassandra_column_infos: List[CassandraColumn]
+    ) -> Generator[SchemaField, None, None]:
+        converter = cls()
+        yield from converter._get_schema_fields(cassandra_column_infos)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
index 86c1c8db11b05..a5eecf198a9b4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
@@ -14,7 +14,6 @@ class DatasetSubTypes(StrEnum):
     ELASTIC_DATASTREAM = "Datastream"
     SALESFORCE_CUSTOM_OBJECT = "Custom Object"
     SALESFORCE_STANDARD_OBJECT = "Object"
-    POWERBI_DATASET_TABLE = "PowerBI Dataset Table"
     QLIK_DATASET = "Qlik Dataset"
     BIGQUERY_TABLE_SNAPSHOT = "Bigquery Table Snapshot"
     SHARDED_TABLE = "Sharded Table"
@@ -23,6 +22,8 @@ class DatasetSubTypes(StrEnum):
     SAC_MODEL = "Model"
     SAC_IMPORT_DATA_MODEL = "Import Data Model"
     SAC_LIVE_DATA_MODEL = "Live Data Model"
+    NEO4J_NODE = "Neo4j Node"
+    NEO4J_RELATIONSHIP = "Neo4j Relationship"
 
     # TODO: Create separate entity...
     NOTEBOOK = "Notebook"
@@ -41,6 +42,7 @@ class DatasetContainerSubTypes(StrEnum):
     S3_BUCKET = "S3 bucket"
     GCS_BUCKET = "GCS bucket"
     ABS_CONTAINER = "ABS container"
+    KEYSPACE = "Keyspace"  # Cassandra
 
 
 class BIContainerSubTypes(StrEnum):
@@ -48,8 +50,8 @@ class BIContainerSubTypes(StrEnum):
     LOOKML_PROJECT = "LookML Project"
     LOOKML_MODEL = "LookML Model"
     TABLEAU_WORKBOOK = "Workbook"
-    POWERBI_WORKSPACE = "Workspace"
-    POWERBI_DATASET = "PowerBI Dataset"
+    POWERBI_DATASET = "Semantic Model"
+    POWERBI_DATASET_TABLE = "Table"
     QLIK_SPACE = "Qlik Space"
     QLIK_APP = "Qlik App"
     SIGMA_WORKSPACE = "Sigma Workspace"
@@ -71,6 +73,7 @@ class BIAssetSubTypes(StrEnum):
     # PowerBI
     POWERBI_TILE = "PowerBI Tile"
     POWERBI_PAGE = "PowerBI Page"
+    POWERBI_APP = "App"
 
     # Mode
     MODE_REPORT = "Report"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py
index 09ce8b5b05203..2b75d0dca53cb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py
@@ -16,8 +16,10 @@
 from datahub.ingestion.extractor import protobuf_util, schema_util
 from datahub.ingestion.extractor.json_schema_util import JsonSchemaTranslator
 from datahub.ingestion.extractor.protobuf_util import ProtobufSchema
-from datahub.ingestion.source.kafka import KafkaSourceConfig, KafkaSourceReport
-from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase
+from datahub.ingestion.source.kafka.kafka import KafkaSourceConfig, KafkaSourceReport
+from datahub.ingestion.source.kafka.kafka_schema_registry_base import (
+    KafkaSchemaRegistryBase,
+)
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     KafkaSchema,
     SchemaField,
@@ -371,7 +373,6 @@ def _get_schema_fields(
     def _get_schema_metadata(
         self, topic: str, platform_urn: str, is_subject: bool
     ) -> Optional[SchemaMetadata]:
-
         # Process the value schema
         schema, fields = self._get_schema_and_fields(
             topic=topic,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py
index e4829f8713cf7..42e025073b534 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py
@@ -653,7 +653,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
             is_resource_row: bool = not row["subresource"]
             entity_urn = row["resource"]
-            entity_type = Urn.create_from_string(row["resource"]).get_type()
+            entity_type = Urn.from_string(row["resource"]).get_type()
 
             term_associations: List[
                 GlossaryTermAssociationClass
diff --git a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/config.py b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/config.py
index 5f88cf0234947..ede7d3c3c5695 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/config.py
@@ -7,7 +7,6 @@
 
 
 class PathSpecsConfigMixin(ConfigModel):
-
     path_specs: List[PathSpec] = Field(
         description="List of PathSpec. See [below](#path-spec) the details about PathSpec"
     )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
index 9705d63912b8d..cd3c2146e6d84 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
@@ -1,5 +1,5 @@
 import os
-from typing import Optional
+from typing import Optional, Set
 
 from pydantic import Field, root_validator
 
@@ -14,6 +14,17 @@
 DEFAULT_DATABASE_TABLE_NAME = "metadata_aspect_v2"
 DEFAULT_KAFKA_TOPIC_NAME = "MetadataChangeLog_Timeseries_v1"
 DEFAULT_DATABASE_BATCH_SIZE = 10_000
+DEFAULT_EXCLUDE_ASPECTS = {
+    "dataHubIngestionSourceKey",
+    "dataHubIngestionSourceInfo",
+    "datahubIngestionRunSummary",
+    "datahubIngestionCheckpoint",
+    "dataHubSecretKey",
+    "dataHubSecretValue",
+    "globalSettingsKey",
+    "globalSettingsInfo",
+    "testResults",
+}
 
 
 class DataHubSourceConfig(StatefulIngestionConfigBase):
@@ -35,6 +46,19 @@ class DataHubSourceConfig(StatefulIngestionConfigBase):
         ),
     )
 
+    include_soft_deleted_entities: bool = Field(
+        default=True,
+        description=(
+            "If enabled, include entities that have been soft deleted. "
+            "Otherwise, include all entities regardless of removal status. "
+        ),
+    )
+
+    exclude_aspects: Set[str] = Field(
+        default=DEFAULT_EXCLUDE_ASPECTS,
+        description="Set of aspect names to exclude from ingestion",
+    )
+
     database_query_batch_size: int = Field(
         default=DEFAULT_DATABASE_BATCH_SIZE,
         description="Number of records to fetch from the database at a time",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
index 6986aac0a7757..382a0d548e38d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
@@ -26,11 +26,17 @@ def __init__(
         self.report = report
         self.graph = graph
 
-    def get_aspects(self) -> Iterable[MetadataChangeProposalWrapper]:
+    def get_urns(self) -> Iterable[str]:
         urns = self.graph.get_urns_by_filter(
-            status=RemovedStatusFilter.ALL,
+            status=RemovedStatusFilter.ALL
+            if self.config.include_soft_deleted_entities
+            else RemovedStatusFilter.NOT_SOFT_DELETED,
             batch_size=self.config.database_query_batch_size,
         )
+        return urns
+
+    def get_aspects(self) -> Iterable[MetadataChangeProposalWrapper]:
+        urns = self.get_urns()
         tasks: List[futures.Future[Iterable[MetadataChangeProposalWrapper]]] = []
         with futures.ThreadPoolExecutor(
             max_workers=self.config.max_workers
@@ -43,6 +49,9 @@ def get_aspects(self) -> Iterable[MetadataChangeProposalWrapper]:
     def _get_aspects_for_urn(self, urn: str) -> Iterable[MetadataChangeProposalWrapper]:
         aspects: Dict[str, _Aspect] = self.graph.get_entity_semityped(urn)  # type: ignore
         for aspect in aspects.values():
+            if aspect.get_aspect_name().lower() in self.config.exclude_aspects:
+                continue
+
             yield MetadataChangeProposalWrapper(
                 entityUrn=urn,
                 aspect=aspect,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
index e4f1bb275487e..80906ca63115f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
@@ -1,11 +1,10 @@
+import contextlib
 import json
 import logging
 from datetime import datetime
-from typing import Any, Generic, Iterable, List, Optional, Tuple, TypeVar
+from typing import Any, Dict, Generic, Iterable, List, Optional, Tuple, TypeVar
 
 from sqlalchemy import create_engine
-from sqlalchemy.engine import Row
-from typing_extensions import Protocol
 
 from datahub.emitter.aspect import ASPECT_MAP
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -21,13 +20,7 @@
 # Should work for at least mysql, mariadb, postgres
 DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
 
-
-class VersionOrderable(Protocol):
-    createdon: Any  # Should restrict to only orderable types
-    version: int
-
-
-ROW = TypeVar("ROW", bound=VersionOrderable)
+ROW = TypeVar("ROW", bound=Dict[str, Any])
 
 
 class VersionOrderer(Generic[ROW]):
@@ -54,14 +47,14 @@ def _process_row(self, row: ROW) -> Iterable[ROW]:
             return
 
         yield from self._attempt_queue_flush(row)
-        if row.version == 0:
+        if row["version"] == 0:
             self._add_to_queue(row)
         else:
             yield row
 
     def _add_to_queue(self, row: ROW) -> None:
         if self.queue is None:
-            self.queue = (row.createdon, [row])
+            self.queue = (row["createdon"], [row])
         else:
             self.queue[1].append(row)
 
@@ -69,7 +62,7 @@ def _attempt_queue_flush(self, row: ROW) -> Iterable[ROW]:
         if self.queue is None:
             return
 
-        if row.createdon > self.queue[0]:
+        if row["createdon"] > self.queue[0]:
             yield from self._flush_queue()
 
     def _flush_queue(self) -> Iterable[ROW]:
@@ -92,6 +85,21 @@ def __init__(
             **connection_config.options,
         )
 
+    @property
+    def soft_deleted_urns_query(self) -> str:
+        return f"""
+            SELECT DISTINCT mav.urn
+            FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)} as mav
+            JOIN (
+                SELECT *,
+                JSON_EXTRACT(metadata, '$.removed') as removed
+                FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)}
+                WHERE aspect = "status" AND version = 0
+            ) as sd ON sd.urn = mav.urn
+            WHERE sd.removed = true
+            ORDER BY mav.urn
+        """
+
     @property
     def query(self) -> str:
         # May repeat rows for the same date
@@ -101,66 +109,137 @@ def query(self) -> str:
         # Relies on createdon order to reflect version order
         # Ordering of entries with the same createdon is handled by VersionOrderer
         return f"""
-            SELECT urn, aspect, metadata, systemmetadata, createdon, version
-            FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)}
-            WHERE createdon >= %(since_createdon)s
-            {"" if self.config.include_all_versions else "AND version = 0"}
-            ORDER BY createdon, urn, aspect, version
-            LIMIT %(limit)s
-            OFFSET %(offset)s
+        SELECT *
+        FROM (
+            SELECT
+                mav.urn,
+                mav.aspect,
+                mav.metadata,
+                mav.systemmetadata,
+                mav.createdon,
+                mav.version,
+                removed
+            FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)} as mav
+            LEFT JOIN (
+                SELECT
+                    *,
+                    JSON_EXTRACT(metadata, '$.removed') as removed
+                FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)}
+                WHERE aspect = 'status'
+                AND version = 0
+            ) as sd ON sd.urn = mav.urn
+            WHERE 1 = 1
+                {"" if self.config.include_all_versions else "AND mav.version = 0"}
+                {"" if not self.config.exclude_aspects else "AND mav.aspect NOT IN %(exclude_aspects)s"}
+                AND mav.createdon >= %(since_createdon)s
+            ORDER BY
+                createdon,
+                urn,
+                aspect,
+                version
+        ) as t
+        WHERE 1=1
+            {"" if self.config.include_soft_deleted_entities else "AND (removed = false or removed is NULL)"}
+        ORDER BY
+            createdon,
+            urn,
+            aspect,
+            version
         """
 
+    def execute_server_cursor(
+        self, query: str, params: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        with self.engine.connect() as conn:
+            if self.engine.dialect.name == "postgresql":
+                with conn.begin():  # Transaction required for PostgreSQL server-side cursor
+                    conn = conn.execution_options(
+                        stream_results=True,
+                        yield_per=self.config.database_query_batch_size,
+                    )
+                    result = conn.execute(query, params)
+                    for row in result:
+                        yield dict(row)
+            elif self.engine.dialect.name == "mysql":  # MySQL
+                import MySQLdb
+
+                with contextlib.closing(
+                    conn.connection.cursor(MySQLdb.cursors.SSCursor)
+                ) as cursor:
+                    logger.debug(f"Using Cursor type: {cursor.__class__.__name__}")
+                    cursor.execute(query, params)
+
+                    columns = [desc[0] for desc in cursor.description]
+                    while True:
+                        rows = cursor.fetchmany(self.config.database_query_batch_size)
+                        if not rows:
+                            break  # Use break instead of return in generator
+                        for row in rows:
+                            yield dict(zip(columns, row))
+            else:
+                raise ValueError(f"Unsupported dialect: {self.engine.dialect.name}")
+
+    def _get_rows(
+        self, from_createdon: datetime, stop_time: datetime
+    ) -> Iterable[Dict[str, Any]]:
+        params = {
+            "exclude_aspects": list(self.config.exclude_aspects),
+            "since_createdon": from_createdon.strftime(DATETIME_FORMAT),
+        }
+        yield from self.execute_server_cursor(self.query, params)
+
     def get_aspects(
         self, from_createdon: datetime, stop_time: datetime
     ) -> Iterable[Tuple[MetadataChangeProposalWrapper, datetime]]:
-        orderer = VersionOrderer[Row](enabled=self.config.include_all_versions)
+        orderer = VersionOrderer[Dict[str, Any]](
+            enabled=self.config.include_all_versions
+        )
         rows = self._get_rows(from_createdon=from_createdon, stop_time=stop_time)
         for row in orderer(rows):
             mcp = self._parse_row(row)
             if mcp:
-                yield mcp, row.createdon
+                yield mcp, row["createdon"]
 
-    def _get_rows(self, from_createdon: datetime, stop_time: datetime) -> Iterable[Row]:
+    def get_soft_deleted_rows(self) -> Iterable[Dict[str, Any]]:
+        """
+        Fetches all soft-deleted entities from the database.
+
+        Yields:
+            Row objects containing URNs of soft-deleted entities
+        """
         with self.engine.connect() as conn:
-            ts = from_createdon
-            offset = 0
-            while ts.timestamp() <= stop_time.timestamp():
-                logger.debug(f"Polling database aspects from {ts}")
-                rows = conn.execute(
-                    self.query,
-                    since_createdon=ts.strftime(DATETIME_FORMAT),
-                    limit=self.config.database_query_batch_size,
-                    offset=offset,
-                )
-                if not rows.rowcount:
-                    return
-
-                for i, row in enumerate(rows):
-                    yield row
-
-                if ts == row.createdon:
-                    offset += i + 1
-                else:
-                    ts = row.createdon
-                    offset = 0
-
-    def _parse_row(self, row: Row) -> Optional[MetadataChangeProposalWrapper]:
+            with contextlib.closing(conn.connection.cursor()) as cursor:
+                logger.debug("Polling soft-deleted urns from database")
+                cursor.execute(self.soft_deleted_urns_query)
+                columns = [desc[0] for desc in cursor.description]
+                while True:
+                    rows = cursor.fetchmany(self.config.database_query_batch_size)
+                    if not rows:
+                        return
+                    for row in rows:
+                        yield dict(zip(columns, row))
+
+    def _parse_row(
+        self, row: Dict[str, Any]
+    ) -> Optional[MetadataChangeProposalWrapper]:
         try:
-            json_aspect = post_json_transform(json.loads(row.metadata))
-            json_metadata = post_json_transform(json.loads(row.systemmetadata or "{}"))
+            json_aspect = post_json_transform(json.loads(row["metadata"]))
+            json_metadata = post_json_transform(
+                json.loads(row["systemmetadata"] or "{}")
+            )
             system_metadata = SystemMetadataClass.from_obj(json_metadata)
             return MetadataChangeProposalWrapper(
-                entityUrn=row.urn,
-                aspect=ASPECT_MAP[row.aspect].from_obj(json_aspect),
+                entityUrn=row["urn"],
+                aspect=ASPECT_MAP[row["aspect"]].from_obj(json_aspect),
                 systemMetadata=system_metadata,
                 changeType=ChangeTypeClass.UPSERT,
             )
         except Exception as e:
             logger.warning(
-                f"Failed to parse metadata for {row.urn}: {e}", exc_info=True
+                f'Failed to parse metadata for {row["urn"]}: {e}', exc_info=True
             )
             self.report.num_database_parse_errors += 1
             self.report.database_parse_errors.setdefault(
                 str(e), LossyDict()
-            ).setdefault(row.aspect, LossyList()).append(row.urn)
+            ).setdefault(row["aspect"], LossyList()).append(row["urn"])
             return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py
index d9e53e87c2cea..56a3d55abb184 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py
@@ -36,6 +36,7 @@ def __init__(
         self.connection_config = connection_config
         self.report = report
         self.group_id = f"{KAFKA_GROUP_PREFIX}-{ctx.pipeline_name}"
+        self.ctx = ctx
 
     def __enter__(self) -> "DataHubKafkaReader":
         self.consumer = DeserializingConsumer(
@@ -95,6 +96,10 @@ def _poll_partition(
                 )
                 break
 
+            if mcl.aspectName and mcl.aspectName in self.config.exclude_aspects:
+                self.report.num_kafka_excluded_aspects += 1
+                continue
+
             # TODO: Consider storing state in kafka instead, via consumer.commit()
             yield mcl, PartitionOffset(partition=msg.partition(), offset=msg.offset())
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
index 0204a864e2b9e..cb72441344088 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
@@ -1,5 +1,5 @@
 import logging
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from functools import partial
 from typing import Dict, Iterable, List, Optional
 
@@ -26,6 +26,7 @@
     StatefulIngestionSourceBase,
 )
 from datahub.metadata.schema_classes import ChangeTypeClass
+from datahub.utilities.progress_timer import ProgressTimer
 
 logger = logging.getLogger(__name__)
 
@@ -62,13 +63,18 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         self.report.stop_time = datetime.now(tz=timezone.utc)
         logger.info(f"Ingesting DataHub metadata up until {self.report.stop_time}")
         state = self.stateful_ingestion_handler.get_last_run_state()
+        database_reader: Optional[DataHubDatabaseReader] = None
 
         if self.config.pull_from_datahub_api:
             yield from self._get_api_workunits()
 
         if self.config.database_connection is not None:
+            database_reader = DataHubDatabaseReader(
+                self.config, self.config.database_connection, self.report
+            )
+
             yield from self._get_database_workunits(
-                from_createdon=state.database_createdon_datetime
+                from_createdon=state.database_createdon_datetime, reader=database_reader
             )
             self._commit_progress()
         else:
@@ -77,7 +83,19 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             )
 
         if self.config.kafka_connection is not None:
-            yield from self._get_kafka_workunits(from_offsets=state.kafka_offsets)
+            soft_deleted_urns = []
+            if not self.config.include_soft_deleted_entities:
+                if database_reader is None:
+                    raise ValueError(
+                        "Cannot exclude soft deleted entities without a database connection"
+                    )
+                soft_deleted_urns = [
+                    row["urn"] for row in database_reader.get_soft_deleted_rows()
+                ]
+
+            yield from self._get_kafka_workunits(
+                from_offsets=state.kafka_offsets, soft_deleted_urns=soft_deleted_urns
+            )
             self._commit_progress()
         else:
             logger.info(
@@ -85,21 +103,20 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             )
 
     def _get_database_workunits(
-        self, from_createdon: datetime
+        self, from_createdon: datetime, reader: DataHubDatabaseReader
     ) -> Iterable[MetadataWorkUnit]:
-        if self.config.database_connection is None:
-            return
-
         logger.info(f"Fetching database aspects starting from {from_createdon}")
-        reader = DataHubDatabaseReader(
-            self.config, self.config.database_connection, self.report
-        )
+        progress = ProgressTimer(report_every=timedelta(seconds=60))
         mcps = reader.get_aspects(from_createdon, self.report.stop_time)
         for i, (mcp, createdon) in enumerate(mcps):
-
             if not self.urn_pattern.allowed(str(mcp.entityUrn)):
                 continue
 
+            if progress.should_report():
+                logger.info(
+                    f"Ingested {i} database aspects so far, currently at {createdon}"
+                )
+
             yield mcp.as_workunit()
             self.report.num_database_aspects_ingested += 1
 
@@ -113,20 +130,29 @@ def _get_database_workunits(
             self._commit_progress(i)
 
     def _get_kafka_workunits(
-        self, from_offsets: Dict[int, int]
+        self, from_offsets: Dict[int, int], soft_deleted_urns: List[str] = []
     ) -> Iterable[MetadataWorkUnit]:
         if self.config.kafka_connection is None:
             return
 
         logger.info("Fetching timeseries aspects from kafka")
         with DataHubKafkaReader(
-            self.config, self.config.kafka_connection, self.report, self.ctx
+            self.config,
+            self.config.kafka_connection,
+            self.report,
+            self.ctx,
         ) as reader:
             mcls = reader.get_mcls(
                 from_offsets=from_offsets, stop_time=self.report.stop_time
             )
             for i, (mcl, offset) in enumerate(mcls):
                 mcp = MetadataChangeProposalWrapper.try_from_mcl(mcl)
+                if mcp.entityUrn in soft_deleted_urns:
+                    self.report.num_timeseries_soft_deleted_aspects_dropped += 1
+                    logger.debug(
+                        f"Dropping soft-deleted aspect of {mcp.aspectName} on {mcp.entityUrn}"
+                    )
+                    continue
                 if mcp.changeType == ChangeTypeClass.DELETE:
                     self.report.num_timeseries_deletions_dropped += 1
                     logger.debug(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/report.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/report.py
index 73e5a798a1553..721fc87989442 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/report.py
@@ -20,6 +20,8 @@ class DataHubSourceReport(StatefulIngestionReport):
 
     num_kafka_aspects_ingested: int = 0
     num_kafka_parse_errors: int = 0
+    num_kafka_excluded_aspects: int = 0
     kafka_parse_errors: LossyDict[str, int] = field(default_factory=LossyDict)
 
     num_timeseries_deletions_dropped: int = 0
+    num_timeseries_soft_deleted_aspects_dropped: int = 0
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
index b39e05a8db4de..66c5ef7179af4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
@@ -70,6 +70,7 @@ def set_metadata_endpoint(cls, values: dict) -> dict:
                 raise ValueError(
                     "Unable to infer the metadata endpoint from the access URL. Please provide a metadata endpoint."
                 )
+            logger.info(f"Inferred metadata endpoint: {metadata_endpoint}")
             values["metadata_endpoint"] = metadata_endpoint
         return values
 
@@ -260,20 +261,6 @@ def infer_metadata_endpoint(access_url: str) -> Optional[str]:
 @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 class DBTCloudSource(DBTSourceBase, TestableSource):
-    """
-    This source pulls dbt metadata directly from the dbt Cloud APIs.
-
-    You'll need to have a dbt Cloud job set up to run your dbt project, and "Generate docs on run" should be enabled.
-
-    The token should have the "read metadata" permission.
-
-    To get the required IDs, go to the job details page (this is the one with the "Run History" table), and look at the URL.
-    It should look something like this: https://cloud.getdbt.com/next/deploy/107298/projects/175705/jobs/148094.
-    In this example, the account ID is 107298, the project ID is 175705, and the job ID is 148094.
-    """
-
-    # TODO: add some screenshots to the docs
-
     config: DBTCloudConfig
 
     @classmethod
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index c15f1deb43a3a..499e7e1231d05 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -53,19 +53,7 @@
     make_assertion_from_test,
     make_assertion_result_from_test,
 )
-from datahub.ingestion.source.sql.sql_types import (
-    ATHENA_SQL_TYPES_MAP,
-    BIGQUERY_TYPES_MAP,
-    POSTGRES_TYPES_MAP,
-    SNOWFLAKE_TYPES_MAP,
-    SPARK_SQL_TYPES_MAP,
-    TRINO_SQL_TYPES_MAP,
-    VERTICA_SQL_TYPES_MAP,
-    resolve_athena_modified_type,
-    resolve_postgres_modified_type,
-    resolve_trino_modified_type,
-    resolve_vertica_modified_type,
-)
+from datahub.ingestion.source.sql.sql_types import resolve_sql_type
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
     StaleEntityRemovalSourceReport,
@@ -89,17 +77,11 @@
 from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
-    BooleanTypeClass,
-    DateTypeClass,
     MySqlDDL,
     NullTypeClass,
-    NumberTypeClass,
-    RecordType,
     SchemaField,
     SchemaFieldDataType,
     SchemaMetadata,
-    StringTypeClass,
-    TimeTypeClass,
 )
 from datahub.metadata.schema_classes import (
     DataPlatformInstanceClass,
@@ -117,9 +99,8 @@
     ViewPropertiesClass,
 )
 from datahub.metadata.urns import DatasetUrn
-from datahub.sql_parsing.schema_resolver import SchemaResolver
+from datahub.sql_parsing.schema_resolver import SchemaInfo, SchemaResolver
 from datahub.sql_parsing.sqlglot_lineage import (
-    SchemaInfo,
     SqlParsingDebugInfo,
     SqlParsingResult,
     infer_output_schema,
@@ -805,28 +786,6 @@ def make_mapping_upstream_lineage(
     )
 
 
-# See https://github.com/fishtown-analytics/dbt/blob/master/core/dbt/adapters/sql/impl.py
-_field_type_mapping = {
-    "boolean": BooleanTypeClass,
-    "date": DateTypeClass,
-    "time": TimeTypeClass,
-    "numeric": NumberTypeClass,
-    "text": StringTypeClass,
-    "timestamp with time zone": DateTypeClass,
-    "timestamp without time zone": DateTypeClass,
-    "integer": NumberTypeClass,
-    "float8": NumberTypeClass,
-    "struct": RecordType,
-    **POSTGRES_TYPES_MAP,
-    **SNOWFLAKE_TYPES_MAP,
-    **BIGQUERY_TYPES_MAP,
-    **SPARK_SQL_TYPES_MAP,
-    **TRINO_SQL_TYPES_MAP,
-    **ATHENA_SQL_TYPES_MAP,
-    **VERTICA_SQL_TYPES_MAP,
-}
-
-
 def get_column_type(
     report: DBTSourceReport,
     dataset_name: str,
@@ -836,24 +795,10 @@ def get_column_type(
     """
     Maps known DBT types to datahub types
     """
-    TypeClass: Any = _field_type_mapping.get(column_type) if column_type else None
-
-    if TypeClass is None and column_type:
-        # resolve a modified type
-        if dbt_adapter == "trino":
-            TypeClass = resolve_trino_modified_type(column_type)
-        elif dbt_adapter == "athena":
-            TypeClass = resolve_athena_modified_type(column_type)
-        elif dbt_adapter == "postgres" or dbt_adapter == "redshift":
-            # Redshift uses a variant of Postgres, so we can use the same logic.
-            TypeClass = resolve_postgres_modified_type(column_type)
-        elif dbt_adapter == "vertica":
-            TypeClass = resolve_vertica_modified_type(column_type)
-        elif dbt_adapter == "snowflake":
-            # Snowflake types are uppercase, so we check that.
-            TypeClass = _field_type_mapping.get(column_type.upper())
-
-    # if still not found, report the warning
+
+    TypeClass = resolve_sql_type(column_type, dbt_adapter)
+
+    # if still not found, report a warning
     if TypeClass is None:
         if column_type:
             report.info(
@@ -862,9 +807,9 @@ def get_column_type(
                 context=f"{dataset_name} - {column_type}",
                 log=False,
             )
-        TypeClass = NullTypeClass
+        TypeClass = NullTypeClass()
 
-    return SchemaFieldDataType(type=TypeClass())
+    return SchemaFieldDataType(type=TypeClass)
 
 
 @platform_name("dbt")
@@ -1074,8 +1019,8 @@ def add_node_to_cll_list(dbt_name: str) -> None:
             for upstream in all_nodes_map[dbt_name].upstream_nodes:
                 schema_nodes.add(upstream)
 
-                upstream_node = all_nodes_map[upstream]
-                if upstream_node.is_ephemeral_model():
+                upstream_node = all_nodes_map.get(upstream)
+                if upstream_node and upstream_node.is_ephemeral_model():
                     add_node_to_cll_list(upstream)
 
             cll_nodes.add(dbt_name)
@@ -1822,16 +1767,21 @@ def _aggregate_owners(
                     logger.debug(
                         f"Owner after applying owner extraction pattern:'{self.config.owner_extraction_pattern}' is '{owner}'."
                     )
-            if self.config.strip_user_ids_from_email:
-                owner = owner.split("@")[0]
-                logger.debug(f"Owner (after stripping email):{owner}")
-
-            owner_list.append(
-                OwnerClass(
-                    owner=mce_builder.make_user_urn(owner),
-                    type=OwnershipTypeClass.DATAOWNER,
+            if isinstance(owner, list):
+                owners = owner
+            else:
+                owners = [owner]
+            for owner in owners:
+                if self.config.strip_user_ids_from_email:
+                    owner = owner.split("@")[0]
+                    logger.debug(f"Owner (after stripping email):{owner}")
+
+                owner_list.append(
+                    OwnerClass(
+                        owner=mce_builder.make_user_urn(owner),
+                        type=OwnershipTypeClass.DATAOWNER,
+                    )
                 )
-            )
 
         owner_list = sorted(owner_list, key=lambda x: x.owner)
         return owner_list
@@ -1989,6 +1939,11 @@ def _translate_dbt_name_to_upstream_urn(dbt_name: str) -> str:
                 time=mce_builder.get_sys_time(),
                 actor=_DEFAULT_ACTOR,
             )
+            sibling_urn = node.get_urn(
+                self.config.target_platform,
+                self.config.env,
+                self.config.target_platform_instance,
+            )
             return UpstreamLineageClass(
                 upstreams=[
                     UpstreamClass(
@@ -1997,6 +1952,7 @@ def _translate_dbt_name_to_upstream_urn(dbt_name: str) -> str:
                         auditStamp=auditStamp,
                     )
                     for upstream in upstream_urns
+                    if not (node.node_type == "model" and upstream == sibling_urn)
                 ],
                 fineGrainedLineages=(
                     (cll or None) if self.config.include_column_lineage else None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py
index 81a54d1327d05..d2b4a576953da 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py
@@ -6,9 +6,8 @@
 from pydantic import Field
 from typing_extensions import Literal
 
-from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
 from datahub.configuration.source_common import (
-    ConfigModel,
     EnvConfigMixin,
     PlatformInstanceConfigMixin,
 )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py
index 6a52d8fdd8905..98133ca69011e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py
@@ -223,15 +223,14 @@ def ingest_table(
         )
 
         customProperties = {
-            "number_of_files": str(get_file_count(delta_table)),
             "partition_columns": str(delta_table.metadata().partition_columns),
             "table_creation_time": str(delta_table.metadata().created_time),
             "id": str(delta_table.metadata().id),
             "version": str(delta_table.version()),
             "location": self.source_config.complete_path,
         }
-        if not self.source_config.require_files:
-            del customProperties["number_of_files"]  # always 0
+        if self.source_config.require_files:
+            customProperties["number_of_files"] = str(get_file_count(delta_table))
 
         dataset_properties = DatasetPropertiesClass(
             description=delta_table.metadata().description,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
new file mode 100644
index 0000000000000..d913b7e42065d
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
@@ -0,0 +1,894 @@
+import concurrent.futures
+import json
+import logging
+import re
+import warnings
+from collections import defaultdict
+from enum import Enum
+from itertools import product
+from time import sleep, time
+from typing import Any, Deque, Dict, List, Optional, Union
+from urllib.parse import quote
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+from urllib3.exceptions import InsecureRequestWarning
+
+from datahub.ingestion.source.dremio.dremio_config import DremioSourceConfig
+from datahub.ingestion.source.dremio.dremio_datahub_source_mapping import (
+    DremioToDataHubSourceTypeMapping,
+)
+from datahub.ingestion.source.dremio.dremio_reporting import DremioSourceReport
+from datahub.ingestion.source.dremio.dremio_sql_queries import DremioSQLQueries
+
+logger = logging.getLogger(__name__)
+
+
+class DremioAPIException(Exception):
+    pass
+
+
+class DremioEdition(Enum):
+    CLOUD = "CLOUD"
+    ENTERPRISE = "ENTERPRISE"
+    COMMUNITY = "COMMUNITY"
+
+
+class DremioEntityContainerType(Enum):
+    SPACE = "SPACE"
+    CONTAINER = "CONTAINER"
+    FOLDER = "FOLDER"
+    SOURCE = "SOURCE"
+
+
+class DremioAPIOperations:
+    _retry_count: int = 5
+    _timeout: int = 1800
+
+    def __init__(
+        self, connection_args: "DremioSourceConfig", report: "DremioSourceReport"
+    ) -> None:
+        self.dremio_to_datahub_source_mapper = DremioToDataHubSourceTypeMapping()
+        self.allow_schema_pattern: List[str] = connection_args.schema_pattern.allow
+        self.deny_schema_pattern: List[str] = connection_args.schema_pattern.deny
+        self._max_workers: int = connection_args.max_workers
+        self.is_dremio_cloud = connection_args.is_dremio_cloud
+        self.report = report
+        self.session = requests.Session()
+        if connection_args.is_dremio_cloud:
+            self.base_url = self._get_cloud_base_url(
+                connection_args,
+            )
+        else:
+            self.base_url = self._get_on_prem_base_url(connection_args)
+
+        self.ui_url = self._get_ui_url(connection_args)
+
+        self.authenticate(connection_args)
+        self.edition = self.get_dremio_edition()
+
+    def get_dremio_edition(self):
+        if self.is_dremio_cloud:
+            return DremioEdition.CLOUD
+        else:
+            return (
+                DremioEdition.ENTERPRISE
+                if self.test_for_enterprise_edition()
+                else DremioEdition.COMMUNITY
+            )
+
+    def _get_cloud_base_url(self, connection_args: "DremioSourceConfig") -> str:
+        """Return the base URL for Dremio Cloud."""
+        if connection_args.dremio_cloud_project_id:
+            project_id = connection_args.dremio_cloud_project_id
+        else:
+            self.report.failure(
+                "Project ID must be provided for Dremio Cloud environments."
+            )
+            raise DremioAPIException(
+                "Project ID must be provided for Dremio Cloud environments."
+            )
+
+        if connection_args.dremio_cloud_region == "US":
+            return f"https://api.dremio.cloud:443/v0/projects/{project_id}"
+        return f"https://api.{connection_args.dremio_cloud_region.lower()}.dremio.cloud:443/v0/projects/{project_id}"
+
+    def _get_on_prem_base_url(self, connection_args: "DremioSourceConfig") -> str:
+        """Return the base URL for on-prem Dremio."""
+        host = connection_args.hostname
+        port = connection_args.port
+        protocol = "https" if connection_args.tls else "http"
+        if not host:
+            self.report.failure(
+                "Hostname must be provided for on-premises Dremio instances."
+            )
+            raise DremioAPIException(
+                "Hostname must be provided for on-premises Dremio instances."
+            )
+        return f"{protocol}://{host}:{port}/api/v3"
+
+    def _get_ui_url(self, connection_args: "DremioSourceConfig") -> str:
+        """Return the UI URL for Dremio."""
+        if connection_args.is_dremio_cloud:
+            if connection_args.dremio_cloud_project_id:
+                project_id = connection_args.dremio_cloud_project_id
+            else:
+                self.report.failure(
+                    "Project ID must be provided for Dremio Cloud environments."
+                )
+                raise DremioAPIException(
+                    "Project ID must be provided for Dremio Cloud environments."
+                )
+            cloud_region = connection_args.dremio_cloud_region
+            if cloud_region == "US":
+                return f"https://app.dremio.cloud/sonar/{project_id}"
+            return f"https://app.{cloud_region.lower()}.dremio.cloud/sonar/{project_id}"
+
+        else:
+            host = connection_args.hostname
+            port = connection_args.port
+            protocol = "https" if connection_args.tls else "http"
+            if not host:
+                self.report.failure(
+                    "Hostname must be provided for on-premises Dremio instances."
+                )
+                raise DremioAPIException(
+                    "Hostname must be provided for on-premises Dremio instances."
+                )
+            return f"{protocol}://{host}:{port}"
+
+    def _setup_session(self) -> None:
+        """Setup the session for retries and connection handling."""
+        retry_strategy = Retry(
+            total=self._retry_count,
+            status_forcelist=[429, 502, 503, 504],
+            allowed_methods=["HEAD", "GET", "OPTIONS", "POST"],
+            backoff_factor=1,
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        self.session.mount("https://", adapter)
+        self.session.mount("http://", adapter)
+        self.session.headers.update({"Content-Type": "application/json"})
+
+    def authenticate(self, connection_args: "DremioSourceConfig") -> None:
+        """Authenticate the session for Dremio, handling both cloud and on-prem cases."""
+        self._setup_session()
+
+        self._verify = (
+            True
+            if connection_args.is_dremio_cloud
+            else (
+                connection_args.tls
+                and not connection_args.disable_certificate_verification
+            )
+        )
+        if not self._verify:
+            warnings.simplefilter("ignore", InsecureRequestWarning)
+
+        # Cloud Dremio authentication using PAT
+        if self.is_dremio_cloud:
+            if not connection_args.password:
+                self.report.failure(
+                    "Personal Access Token (PAT) is missing for cloud authentication."
+                )
+                raise DremioAPIException(
+                    "Personal Access Token (PAT) is missing for cloud authentication."
+                )
+            self.session.headers.update(
+                {"Authorization": f"Bearer {connection_args.password}"}
+            )
+            return
+
+        # On-prem Dremio authentication (PAT or Basic Auth)
+        for retry in range(1, self._retry_count + 1):
+            try:
+                if connection_args.authentication_method == "PAT":
+                    self.session.headers.update(
+                        {
+                            "Authorization": f"Bearer {connection_args.password}",
+                        }
+                    )
+                    return
+                else:
+                    assert (
+                        connection_args.username and connection_args.password
+                    ), "Username and password are required for authentication"
+                    host = connection_args.hostname
+                    port = connection_args.port
+                    protocol = "https" if connection_args.tls else "http"
+                    login_url = f"{protocol}://{host}:{port}/apiv2/login"
+                    response = self.session.post(
+                        url=login_url,
+                        data=json.dumps(
+                            {
+                                "userName": connection_args.username,
+                                "password": connection_args.password,
+                            }
+                        ),
+                        verify=self._verify,
+                        timeout=self._timeout,
+                    )
+                    response.raise_for_status()
+                    token = response.json().get("token")
+                    if token:
+                        self.session.headers.update(
+                            {"Authorization": f"_dremio{token}"}
+                        )
+
+                        return
+                    else:
+                        self.report.failure("Failed to authenticate", login_url)
+                        raise DremioAPIException("Failed to authenticate with Dremio")
+            except Exception as e:
+                self.report.failure("Failed to authenticate", str(e))
+                sleep(1)  # Optional: exponential backoff
+
+        self.report.failure(
+            "Credentials cannot be refreshed. Please check your username and password."
+        )
+        raise DremioAPIException(
+            "Credentials cannot be refreshed. Please check your username and password."
+        )
+
+    def get(self, url: str) -> Dict:
+        """execute a get request on dremio"""
+        response = self.session.get(
+            url=(self.base_url + url),
+            verify=self._verify,
+            timeout=self._timeout,
+        )
+        return response.json()
+
+    def post(self, url: str, data: str) -> Dict:
+        """execute a get request on dremio"""
+        response = self.session.post(
+            url=(self.base_url + url),
+            data=data,
+            verify=self._verify,
+            timeout=self._timeout,
+        )
+        return response.json()
+
+    def execute_query(self, query: str, timeout: int = 3600) -> List[Dict[str, Any]]:
+        """Execute SQL query with timeout and error handling"""
+        try:
+            response = self.post(url="/sql", data=json.dumps({"sql": query}))
+
+            if "errorMessage" in response:
+                self.report.failure(
+                    message="SQL Error", context=f"{response['errorMessage']}"
+                )
+                raise DremioAPIException(f"SQL Error: {response['errorMessage']}")
+
+            job_id = response["id"]
+
+            with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+                future = executor.submit(self.fetch_results, job_id)
+                try:
+                    return future.result(timeout=timeout)
+                except concurrent.futures.TimeoutError:
+                    self.cancel_query(job_id)
+                    raise DremioAPIException(
+                        f"Query execution timed out after {timeout} seconds"
+                    )
+                except RuntimeError as e:
+                    raise DremioAPIException(f"{str(e)}")
+
+        except requests.RequestException as e:
+            raise DremioAPIException(f"Error executing query: {str(e)}")
+
+    def fetch_results(self, job_id: str) -> List[Dict]:
+        """Fetch job results with status checking"""
+        start_time = time()
+        while True:
+            status = self.get_job_status(job_id)
+            if status["jobState"] == "COMPLETED":
+                break
+            elif status["jobState"] == "FAILED":
+                error_message = status.get("errorMessage", "Unknown error")
+                raise RuntimeError(f"Query failed: {error_message}")
+            elif status["jobState"] == "CANCELED":
+                raise RuntimeError("Query was canceled")
+
+            if time() - start_time > self._timeout:
+                self.cancel_query(job_id)
+                raise TimeoutError("Query execution timed out while fetching results")
+
+            sleep(3)
+
+        return self._fetch_all_results(job_id)
+
+    def _fetch_all_results(self, job_id: str) -> List[Dict]:
+        """Fetch all results for a completed job"""
+        limit = 500
+        offset = 0
+        rows = []
+
+        while True:
+            result = self.get_job_result(job_id, offset, limit)
+            rows.extend(result["rows"])
+
+            offset = offset + limit
+            if offset >= result["rowCount"]:
+                break
+
+        return rows
+
+    def cancel_query(self, job_id: str) -> None:
+        """Cancel a running query"""
+        try:
+            self.post(url=f"/job/{job_id}/cancel", data=json.dumps({}))
+        except Exception as e:
+            logger.error(f"Failed to cancel query {job_id}: {str(e)}")
+
+    def get_job_status(self, job_id: str) -> Dict[str, Any]:
+        """Check job status"""
+        return self.get(
+            url=f"/job/{job_id}/",
+        )
+
+    def get_job_result(
+        self, job_id: str, offset: int = 0, limit: int = 500
+    ) -> Dict[str, Any]:
+        """Get job results in batches"""
+        return self.get(
+            url=f"/job/{job_id}/results?offset={offset}&limit={limit}",
+        )
+
+    def get_dataset_id(self, schema: str, dataset: str) -> Optional[str]:
+        """Retrieve the dataset ID based on schema and dataset name."""
+        schema_split = schema.split(".")
+        schema_str = ""
+        last_val = 0
+
+        for increment_val in range(1, len(schema_split) + 1):
+            current_path = ".".join(schema_split[last_val:increment_val])
+            url_encoded = quote(current_path, safe="")
+            response = self.get(url=f"/catalog/by-path/{schema_str}/{url_encoded}")
+
+            if not response.get("errorMessage"):
+                last_val = increment_val
+                schema_str = (
+                    f"{schema_str}/{url_encoded}" if schema_str else url_encoded
+                )
+
+        dataset_response = self.get(
+            url=f"/catalog/by-path/{schema_str}/{quote(dataset, safe='')}",
+        )
+        dataset_id = dataset_response.get("id")
+        if not dataset_id:
+            logger.error(f"Dataset ID not found for {schema}.{dataset}")
+
+        return dataset_id
+
+    def community_get_formatted_tables(
+        self, tables_and_columns: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        schema_list = []
+        schema_dict_lookup = []
+        dataset_list = []
+        column_dictionary: Dict[str, List[Dict]] = defaultdict(list)
+
+        ordinal_position = 0
+        for record in tables_and_columns:
+            if not record.get("COLUMN_NAME"):
+                continue
+
+            table_full_path = record.get("FULL_TABLE_PATH")
+            if not table_full_path:
+                continue
+
+            column_dictionary[table_full_path].append(
+                {
+                    "name": record["COLUMN_NAME"],
+                    "ordinal_position": record.get(
+                        "ORDINAL_POSITION", ordinal_position
+                    ),
+                    "is_nullable": record["IS_NULLABLE"],
+                    "data_type": record["DATA_TYPE"],
+                    "column_size": record["COLUMN_SIZE"],
+                }
+            )
+
+            ordinal_position += 1
+
+            if record.get("TABLE_SCHEMA") not in schema_list:
+                schema_list.append(record.get("TABLE_SCHEMA"))
+
+        distinct_tables_list = list(
+            {
+                tuple(
+                    dictionary[key]
+                    for key in (
+                        "TABLE_SCHEMA",
+                        "TABLE_NAME",
+                        "FULL_TABLE_PATH",
+                        "VIEW_DEFINITION",
+                    )
+                    if key in dictionary
+                ): dictionary
+                for dictionary in tables_and_columns
+            }.values()
+        )
+
+        for schema in schema_list:
+            schema_dict_lookup.append(self.validate_schema_format(schema))
+
+        for table, schemas in product(distinct_tables_list, schema_dict_lookup):
+            if table.get("TABLE_SCHEMA") == schemas.get("original_path"):
+                dataset_list.append(
+                    {
+                        "TABLE_SCHEMA": "["
+                        + ", ".join(
+                            schemas.get("formatted_path") + [table.get("TABLE_NAME")]
+                        )
+                        + "]",
+                        "TABLE_NAME": table.get("TABLE_NAME"),
+                        "COLUMNS": column_dictionary.get(
+                            table.get("FULL_TABLE_PATH", "")
+                        ),
+                        "VIEW_DEFINITION": table.get("VIEW_DEFINITION"),
+                        "RESOURCE_ID": self.get_dataset_id(
+                            schema=".".join(schemas.get("formatted_path")),
+                            dataset=table.get("TABLE_NAME", ""),
+                        ),
+                        "LOCATION_ID": self.get_dataset_id(
+                            schema=".".join(schemas.get("formatted_path")),
+                            dataset="",
+                        ),
+                    }
+                )
+
+        return dataset_list
+
+    def get_pattern_condition(
+        self, patterns: Union[str, List[str]], field: str, allow: bool = True
+    ) -> str:
+        if not patterns:
+            return ""
+
+        if isinstance(patterns, str):
+            patterns = [patterns.upper()]
+
+        if ".*" in patterns and allow:
+            return ""
+
+        patterns = [p.upper() for p in patterns if p != ".*"]
+        if not patterns:
+            return ""
+
+        operator = "REGEXP_LIKE" if allow else "NOT REGEXP_LIKE"
+        pattern_str = "|".join(f"({p})" for p in patterns)
+        return f"AND {operator}({field}, '{pattern_str}')"
+
+    def get_all_tables_and_columns(self, containers: Deque) -> List[Dict]:
+        if self.edition == DremioEdition.ENTERPRISE:
+            query_template = DremioSQLQueries.QUERY_DATASETS_EE
+        elif self.edition == DremioEdition.CLOUD:
+            query_template = DremioSQLQueries.QUERY_DATASETS_CLOUD
+        else:
+            query_template = DremioSQLQueries.QUERY_DATASETS_CE
+
+        schema_field = "CONCAT(REPLACE(REPLACE(REPLACE(UPPER(TABLE_SCHEMA), ', ', '.'), '[', ''), ']', ''))"
+
+        schema_condition = self.get_pattern_condition(
+            self.allow_schema_pattern, schema_field
+        )
+        deny_schema_condition = self.get_pattern_condition(
+            self.deny_schema_pattern, schema_field, allow=False
+        )
+
+        all_tables_and_columns = []
+
+        for schema in containers:
+            formatted_query = ""
+            try:
+                formatted_query = query_template.format(
+                    schema_pattern=schema_condition,
+                    deny_schema_pattern=deny_schema_condition,
+                    container_name=schema.container_name.lower(),
+                )
+                all_tables_and_columns.extend(
+                    self.execute_query(
+                        query=formatted_query,
+                    )
+                )
+            except DremioAPIException as e:
+                self.report.warning(
+                    message="Container has no tables or views",
+                    context=f"{schema.subclass} {schema.container_name}",
+                    exc=e,
+                )
+
+        tables = []
+
+        if self.edition == DremioEdition.COMMUNITY:
+            tables = self.community_get_formatted_tables(all_tables_and_columns)
+
+        else:
+            column_dictionary: Dict[str, List[Dict]] = defaultdict(list)
+
+            for record in all_tables_and_columns:
+                if not record.get("COLUMN_NAME"):
+                    continue
+
+                table_full_path = record.get("FULL_TABLE_PATH")
+                if not table_full_path:
+                    continue
+
+                column_dictionary[table_full_path].append(
+                    {
+                        "name": record["COLUMN_NAME"],
+                        "ordinal_position": record["ORDINAL_POSITION"],
+                        "is_nullable": record["IS_NULLABLE"],
+                        "data_type": record["DATA_TYPE"],
+                        "column_size": record["COLUMN_SIZE"],
+                    }
+                )
+
+            distinct_tables_list = list(
+                {
+                    tuple(
+                        dictionary[key]
+                        for key in (
+                            "TABLE_SCHEMA",
+                            "TABLE_NAME",
+                            "FULL_TABLE_PATH",
+                            "VIEW_DEFINITION",
+                            "LOCATION_ID",
+                            "OWNER",
+                            "OWNER_TYPE",
+                            "CREATED",
+                            "FORMAT_TYPE",
+                        )
+                        if key in dictionary
+                    ): dictionary
+                    for dictionary in all_tables_and_columns
+                }.values()
+            )
+
+            for table in distinct_tables_list:
+                tables.append(
+                    {
+                        "TABLE_NAME": table.get("TABLE_NAME"),
+                        "TABLE_SCHEMA": table.get("TABLE_SCHEMA"),
+                        "COLUMNS": column_dictionary[table["FULL_TABLE_PATH"]],
+                        "VIEW_DEFINITION": table.get("VIEW_DEFINITION"),
+                        "RESOURCE_ID": table.get("RESOURCE_ID"),
+                        "LOCATION_ID": table.get("LOCATION_ID"),
+                        "OWNER": table.get("OWNER"),
+                        "OWNER_TYPE": table.get("OWNER_TYPE"),
+                        "CREATED": table.get("CREATED"),
+                        "FORMAT_TYPE": table.get("FORMAT_TYPE"),
+                    }
+                )
+
+        return tables
+
+    def validate_schema_format(self, schema):
+        if "." in schema:
+            schema_path = self.get(
+                url=f"/catalog/{self.get_dataset_id(schema=schema, dataset='')}"
+            ).get("path")
+            return {"original_path": schema, "formatted_path": schema_path}
+        return {"original_path": schema, "formatted_path": [schema]}
+
+    def test_for_enterprise_edition(self):
+        response = self.session.get(
+            url=f"{self.base_url}/catalog/privileges",
+            verify=self._verify,
+            timeout=self._timeout,
+        )
+
+        if response.status_code == 200:
+            return True
+
+        return False
+
+    def get_view_parents(self, dataset_id: str) -> List:
+        parents_list = []
+
+        if self.edition == DremioEdition.ENTERPRISE:
+            parents = self.get(
+                url=f"/catalog/{dataset_id}/graph",
+            ).get("parents")
+
+            if not parents:
+                return []
+
+            for parent in parents:
+                parents_list.append(".".join(parent.get("path")))
+
+        return parents_list
+
+    def extract_all_queries(self) -> List[Dict[str, Any]]:
+        if self.edition == DremioEdition.CLOUD:
+            jobs_query = DremioSQLQueries.QUERY_ALL_JOBS_CLOUD
+        else:
+            jobs_query = DremioSQLQueries.QUERY_ALL_JOBS
+
+        return self.execute_query(query=jobs_query)
+
+    def get_tags_for_resource(self, resource_id: str) -> Optional[List[str]]:
+        """
+        Get Dremio tags for a given resource_id.
+        """
+
+        try:
+            tags = self.get(
+                url=f"/catalog/{resource_id}/collaboration/tag",
+            )
+            return tags.get("tags")
+        except Exception as exc:
+            logging.info(
+                "Resource ID {} has no tags: {}".format(
+                    resource_id,
+                    exc,
+                )
+            )
+        return None
+
+    def get_description_for_resource(self, resource_id: str) -> Optional[str]:
+        """
+        Get Dremio wiki entry for a given resource_id.
+        """
+
+        try:
+            tags = self.get(
+                url=f"/catalog/{resource_id}/collaboration/wiki",
+            )
+            return tags.get("text")
+        except Exception as exc:
+            logging.info(
+                "Resource ID {} has no wiki entry: {}".format(
+                    resource_id,
+                    exc,
+                )
+            )
+        return None
+
+    def _check_pattern_match(
+        self,
+        pattern: str,
+        paths: List[str],
+        allow_prefix: bool = True,
+    ) -> bool:
+        """
+        Helper method to check if a pattern matches any of the paths.
+        Handles hierarchical matching where each level is matched independently.
+        Also handles prefix matching for partial paths.
+        """
+        if pattern == ".*":
+            return True
+
+        # Convert the pattern to regex with proper anchoring
+        regex_pattern = pattern
+        if pattern.startswith("^"):
+            # Already has start anchor
+            regex_pattern = pattern.replace(".", r"\.")  # Escape dots
+            regex_pattern = regex_pattern.replace(
+                r"\.*", ".*"
+            )  # Convert .* to wildcard
+        else:
+            # Add start anchor and handle dots
+            regex_pattern = "^" + pattern.replace(".", r"\.").replace(r"\.*", ".*")
+
+        # Handle end matching
+        if not pattern.endswith(".*"):
+            if pattern.endswith("$"):
+                # Keep explicit end anchor
+                pass
+            elif not allow_prefix:
+                # Add end anchor for exact matching
+                regex_pattern = regex_pattern + "$"
+
+        for path in paths:
+            if re.match(regex_pattern, path, re.IGNORECASE):
+                return True
+
+        return False
+
+    def should_include_container(self, path: List[str], name: str) -> bool:
+        """
+        Helper method to check if a container should be included based on schema patterns.
+        Used by both get_all_containers and get_containers_for_location.
+        """
+        path_components = path + [name] if path else [name]
+        full_path = ".".join(path_components)
+
+        # Default allow everything case
+        if self.allow_schema_pattern == [".*"] and not self.deny_schema_pattern:
+            self.report.report_container_scanned(full_path)
+            return True
+
+        # Check deny patterns first
+        if self.deny_schema_pattern:
+            for pattern in self.deny_schema_pattern:
+                if self._check_pattern_match(
+                    pattern=pattern,
+                    paths=[full_path],
+                    allow_prefix=False,
+                ):
+                    self.report.report_container_filtered(full_path)
+                    return False
+
+        # Check allow patterns
+        for pattern in self.allow_schema_pattern:
+            # For patterns with wildcards, check if this path is a parent of the pattern
+            if "*" in pattern:
+                pattern_parts = pattern.split(".")
+                path_parts = path_components
+
+                # If pattern has exact same number of parts, check each component
+                if len(pattern_parts) == len(path_parts):
+                    matches = True
+                    for p_part, c_part in zip(pattern_parts, path_parts):
+                        if p_part != "*" and p_part.lower() != c_part.lower():
+                            matches = False
+                            break
+                    if matches:
+                        self.report.report_container_scanned(full_path)
+                        return True
+                # Otherwise check if current path is prefix match
+                else:
+                    # Remove the trailing wildcard if present
+                    if pattern_parts[-1] == "*":
+                        pattern_parts = pattern_parts[:-1]
+
+                    for i in range(len(path_parts)):
+                        current_path = ".".join(path_parts[: i + 1])
+                        pattern_prefix = ".".join(pattern_parts[: i + 1])
+
+                        if pattern_prefix.startswith(current_path):
+                            self.report.report_container_scanned(full_path)
+                            return True
+
+            # Direct pattern matching
+            if self._check_pattern_match(
+                pattern=pattern,
+                paths=[full_path],
+                allow_prefix=True,
+            ):
+                self.report.report_container_scanned(full_path)
+                return True
+
+        self.report.report_container_filtered(full_path)
+        return False
+
+    def get_all_containers(self):
+        """
+        Query the Dremio sources API and return filtered source information.
+        """
+        containers = []
+        response = self.get(url="/catalog")
+
+        def process_source(source):
+            if source.get("containerType") == DremioEntityContainerType.SOURCE.value:
+                source_resp = self.get(
+                    url=f"/catalog/{source.get('id')}",
+                )
+
+                source_config = source_resp.get("config", {})
+                db = source_config.get(
+                    "database", source_config.get("databaseName", "")
+                )
+
+                if self.should_include_container([], source.get("path")[0]):
+                    return {
+                        "id": source.get("id"),
+                        "name": source.get("path")[0],
+                        "path": [],
+                        "container_type": DremioEntityContainerType.SOURCE,
+                        "source_type": source_resp.get("type"),
+                        "root_path": source_config.get("rootPath"),
+                        "database_name": db,
+                    }
+            else:
+                if self.should_include_container([], source.get("path")[0]):
+                    return {
+                        "id": source.get("id"),
+                        "name": source.get("path")[0],
+                        "path": [],
+                        "container_type": DremioEntityContainerType.SPACE,
+                    }
+            return None
+
+        def process_source_and_containers(source):
+            container = process_source(source)
+            if not container:
+                return []
+
+            # Get sub-containers
+            sub_containers = self.get_containers_for_location(
+                resource_id=container.get("id"),
+                path=[container.get("name")],
+            )
+
+            return [container] + sub_containers
+
+        # Use ThreadPoolExecutor to parallelize the processing of sources
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=self._max_workers
+        ) as executor:
+            future_to_source = {
+                executor.submit(process_source_and_containers, source): source
+                for source in response.get("data", [])
+            }
+
+            for future in concurrent.futures.as_completed(future_to_source):
+                source = future_to_source[future]
+                try:
+                    containers.extend(future.result())
+                except Exception as exc:
+                    logger.error(f"Error processing source: {exc}")
+                    self.report.warning(
+                        message="Failed to process source",
+                        context=f"{source}",
+                        exc=exc,
+                    )
+
+        return containers
+
+    def get_context_for_vds(self, resource_id: str) -> str:
+        context_array = self.get(
+            url=f"/catalog/{resource_id}",
+        ).get("sqlContext")
+        if context_array:
+            return ".".join(
+                f'"{part}"' if "." in part else f"{part}" for part in context_array
+            )
+        else:
+            return ""
+
+    def get_containers_for_location(
+        self, resource_id: str, path: List[str]
+    ) -> List[Dict[str, str]]:
+        containers = []
+
+        def traverse_path(location_id: str, entity_path: List[str]) -> List:
+            nonlocal containers
+            try:
+                response = self.get(url=f"/catalog/{location_id}")
+
+                # Check if current folder should be included
+                if (
+                    response.get("entityType")
+                    == DremioEntityContainerType.FOLDER.value.lower()
+                ):
+                    folder_name = entity_path[-1]
+                    folder_path = entity_path[:-1]
+
+                    if self.should_include_container(folder_path, folder_name):
+                        containers.append(
+                            {
+                                "id": location_id,
+                                "name": folder_name,
+                                "path": folder_path,
+                                "container_type": DremioEntityContainerType.FOLDER,
+                            }
+                        )
+
+                # Recursively process child containers
+                for container in response.get("children", []):
+                    if (
+                        container.get("type")
+                        == DremioEntityContainerType.CONTAINER.value
+                    ):
+                        traverse_path(container.get("id"), container.get("path"))
+
+            except Exception as exc:
+                logging.info(
+                    "Location {} contains no tables or views. Skipping...".format(
+                        location_id
+                    )
+                )
+                self.report.warning(
+                    message="Failed to get tables or views",
+                    context=f"{location_id}",
+                    exc=exc,
+                )
+
+            return containers
+
+        return traverse_path(location_id=resource_id, entity_path=path)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py
new file mode 100644
index 0000000000000..d9d85edbf4f7a
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py
@@ -0,0 +1,525 @@
+import logging
+import time
+import uuid
+from datetime import datetime
+from typing import Dict, Iterable, List, Optional, Tuple, Type, Union
+
+from datahub._codegen.aspect import _Aspect
+from datahub.emitter.mce_builder import (
+    make_dataplatform_instance_urn,
+    make_domain_urn,
+    make_group_urn,
+    make_user_urn,
+)
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.mcp_builder import ContainerKey
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.dremio.dremio_entities import (
+    DremioContainer,
+    DremioDataset,
+    DremioDatasetColumn,
+    DremioDatasetType,
+    DremioGlossaryTerm,
+)
+from datahub.metadata.schema_classes import (
+    ArrayTypeClass,
+    AuditStampClass,
+    BooleanTypeClass,
+    BrowsePathEntryClass,
+    BrowsePathsV2Class,
+    BytesTypeClass,
+    ContainerClass,
+    ContainerPropertiesClass,
+    DataPlatformInstanceClass,
+    DatasetFieldProfileClass,
+    DatasetProfileClass,
+    DatasetPropertiesClass,
+    DateTypeClass,
+    DomainsClass,
+    GlossaryTermAssociationClass,
+    GlossaryTermInfoClass,
+    GlossaryTermsClass,
+    MySqlDDLClass,
+    NullTypeClass,
+    NumberTypeClass,
+    OwnerClass,
+    OwnershipClass,
+    OwnershipTypeClass,
+    QuantileClass,
+    RecordTypeClass,
+    SchemaFieldClass,
+    SchemaFieldDataTypeClass,
+    SchemaMetadataClass,
+    StatusClass,
+    StringTypeClass,
+    SubTypesClass,
+    TimeStampClass,
+    TimeTypeClass,
+    ViewPropertiesClass,
+)
+
+logger = logging.getLogger(__name__)
+namespace = uuid.NAMESPACE_DNS
+
+
+class DremioContainerKey(ContainerKey):
+    key: str
+
+
+class SchemaFieldTypeMapper:
+    # From https://docs.dremio.com/cloud/reference/sql/data-types/
+
+    FIELD_TYPE_MAPPING: Dict[str, Type] = {
+        # Bool
+        "boolean": BooleanTypeClass,
+        # Binary
+        "binary varying": BytesTypeClass,
+        # Numbers
+        "decimal": NumberTypeClass,
+        "integer": NumberTypeClass,
+        "bigint": NumberTypeClass,
+        "float": NumberTypeClass,
+        "double": NumberTypeClass,
+        # Dates and times
+        "timestamp": DateTypeClass,
+        "date": DateTypeClass,
+        "time": TimeTypeClass,
+        # Strings
+        "char": StringTypeClass,
+        "character": StringTypeClass,
+        "character varying": StringTypeClass,
+        # Records
+        "row": RecordTypeClass,
+        "struct": RecordTypeClass,
+        "list": RecordTypeClass,
+        "map": RecordTypeClass,
+        # Arrays
+        "array": ArrayTypeClass,
+    }
+
+    @classmethod
+    def get_field_type(
+        cls, data_type: str, data_size: Optional[int] = None
+    ) -> Tuple["SchemaFieldDataTypeClass", str]:
+        """
+        Maps a Dremio data type and size to a DataHub SchemaFieldDataTypeClass and native data type string.
+
+        :param data_type: The data type string from Dremio.
+        :param data_size: The size of the data type, if applicable.
+        :return: A tuple containing a SchemaFieldDataTypeClass instance and the native data type string.
+        """
+        if not data_type:
+            logger.warning("Empty data_type provided, defaulting to NullTypeClass.")
+            type_class = NullTypeClass
+            native_data_type = "NULL"
+        else:
+            data_type = data_type.lower()
+            type_class = cls.FIELD_TYPE_MAPPING.get(data_type, NullTypeClass)
+
+            if data_size:
+                native_data_type = f"{data_type}({data_size})"
+            else:
+                native_data_type = data_type
+
+        try:
+            schema_field_type = SchemaFieldDataTypeClass(type=type_class())
+            logger.debug(
+                f"Mapped data_type '{data_type}' with size '{data_size}' to type class "
+                f"'{type_class.__name__}' and native data type '{native_data_type}'."
+            )
+        except Exception as e:
+            logger.error(
+                f"Error initializing SchemaFieldDataTypeClass with type '{type_class.__name__}': {e}"
+            )
+            schema_field_type = SchemaFieldDataTypeClass(type=NullTypeClass())
+
+        return schema_field_type, native_data_type
+
+
+class DremioAspects:
+    def __init__(
+        self,
+        platform: str,
+        ui_url: str,
+        env: str,
+        ingest_owner: bool,
+        domain: Optional[str] = None,
+        platform_instance: Optional[str] = None,
+    ):
+        self.platform = platform
+        self.platform_instance = platform_instance
+        self.env = env
+        self.domain = domain
+        self.ui_url = ui_url
+        self.ingest_owner = ingest_owner
+
+    def get_container_key(
+        self, name: Optional[str], path: Optional[List[str]]
+    ) -> DremioContainerKey:
+        key = name
+        if path:
+            key = ".".join(path) + "." + name if name else ".".join(path)
+
+        return DremioContainerKey(
+            platform=self.platform,
+            instance=self.platform_instance,
+            env=str(self.env),
+            key=key,
+        )
+
+    def get_container_urn(
+        self, name: Optional[str] = None, path: Optional[List[str]] = []
+    ) -> str:
+        container_key = self.get_container_key(name, path)
+        return container_key.as_urn()
+
+    def create_domain_aspect(self) -> Optional[_Aspect]:
+        if self.domain:
+            if self.domain.startswith("urn:li:domain:"):
+                return DomainsClass(domains=[self.domain])
+            return DomainsClass(
+                domains=[
+                    make_domain_urn(
+                        str(uuid.uuid5(namespace, self.domain)),
+                    )
+                ]
+            )
+        return None
+
+    def populate_container_mcp(
+        self, container_urn: str, container: DremioContainer
+    ) -> Iterable[MetadataWorkUnit]:
+        # Container Properties
+        container_properties = self._create_container_properties(container)
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=container_urn,
+            aspect=container_properties,
+            aspectName=ContainerPropertiesClass.ASPECT_NAME,
+        )
+        yield mcp.as_workunit()
+
+        if not container.path:
+            browse_paths_v2 = self._create_browse_paths_containers(container)
+            if browse_paths_v2:
+                mcp = MetadataChangeProposalWrapper(
+                    entityUrn=container_urn,
+                    aspect=browse_paths_v2,
+                )
+                yield mcp.as_workunit()
+
+        # Container Class Folders
+        container_class = self._create_container_class(container)
+        if container_class:
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=container_urn,
+                aspect=container_class,
+            )
+            yield mcp.as_workunit()
+
+        # Data Platform Instance
+        data_platform_instance = self._create_data_platform_instance()
+        if data_platform_instance:
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=container_urn,
+                aspect=data_platform_instance,
+            )
+            yield mcp.as_workunit()
+
+        # SubTypes
+        subtypes = SubTypesClass(typeNames=[container.subclass])
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=container_urn,
+            aspect=subtypes,
+        )
+        yield mcp.as_workunit()
+
+        # Status
+        status = StatusClass(removed=False)
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=container_urn,
+            aspect=status,
+        )
+        yield mcp.as_workunit()
+
+    def populate_dataset_mcp(
+        self, dataset_urn: str, dataset: DremioDataset
+    ) -> Iterable[MetadataWorkUnit]:
+        # Dataset Properties
+        dataset_properties = self._create_dataset_properties(dataset)
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=dataset_urn,
+            aspect=dataset_properties,
+        )
+        yield mcp.as_workunit()
+
+        # Ownership
+        ownership = self._create_ownership(dataset)
+        if ownership:
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=ownership,
+            )
+            yield mcp.as_workunit()
+
+        # SubTypes
+        subtypes = SubTypesClass(typeNames=[dataset.dataset_type.value])
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=dataset_urn,
+            aspect=subtypes,
+        )
+        yield mcp.as_workunit()
+
+        # Data Platform Instance
+        data_platform_instance = self._create_data_platform_instance()
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=dataset_urn,
+            aspect=data_platform_instance,
+        )
+        yield mcp.as_workunit()
+
+        # Container Class
+        container_class = self._create_container_class(dataset)
+        if container_class:
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=container_class,
+            )
+            yield mcp.as_workunit()
+
+        # View Definition
+        if dataset.dataset_type == DremioDatasetType.VIEW:
+            view_definition = self._create_view_properties(dataset)
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=view_definition,
+            )
+            yield mcp.as_workunit()
+
+        # Glossary Terms
+        if dataset.glossary_terms:
+            glossary_terms = self._create_glossary_terms(dataset)
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=glossary_terms,
+            )
+            yield mcp.as_workunit()
+
+        # Schema Metadata
+        if dataset.columns:
+            schema_metadata = self._create_schema_metadata(dataset)
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=schema_metadata,
+            )
+            yield mcp.as_workunit()
+
+        else:
+            logger.warning(
+                f"Dataset {dataset.path}.{dataset.resource_name} has not been queried in Dremio"
+            )
+            logger.warning(
+                f"Dataset {dataset.path}.{dataset.resource_name} will have a null schema"
+            )
+
+        # Status
+        status = StatusClass(removed=False)
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=dataset_urn,
+            aspect=status,
+        )
+        yield mcp.as_workunit()
+
+    def populate_glossary_term_mcp(
+        self, glossary_term: DremioGlossaryTerm
+    ) -> Iterable[MetadataWorkUnit]:
+        glossary_term_info = self._create_glossary_term_info(glossary_term)
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=glossary_term.urn,
+            aspect=glossary_term_info,
+        )
+        yield mcp.as_workunit()
+
+    def populate_profile_aspect(self, profile_data: Dict) -> DatasetProfileClass:
+        field_profiles = [
+            self._create_field_profile(field_name, field_stats)
+            for field_name, field_stats in profile_data.get("column_stats", {}).items()
+        ]
+        return DatasetProfileClass(
+            timestampMillis=round(time.time() * 1000),
+            rowCount=profile_data.get("row_count"),
+            columnCount=profile_data.get("column_count"),
+            fieldProfiles=field_profiles,
+        )
+
+    def _create_container_properties(
+        self, container: DremioContainer
+    ) -> ContainerPropertiesClass:
+        return ContainerPropertiesClass(
+            name=container.container_name,
+            qualifiedName=f"{'.'.join(container.path) + '.' if container.path else ''}{container.container_name}",
+            description=container.description,
+            env=self.env,
+        )
+
+    def _create_browse_paths_containers(
+        self, entity: DremioContainer
+    ) -> Optional[BrowsePathsV2Class]:
+        paths = []
+
+        if entity.subclass == "Dremio Space":
+            paths.append(BrowsePathEntryClass(id="Spaces"))
+        elif entity.subclass == "Dremio Source":
+            paths.append(BrowsePathEntryClass(id="Sources"))
+        if paths:
+            return BrowsePathsV2Class(path=paths)
+        return None
+
+    def _create_container_class(
+        self, entity: Union[DremioContainer, DremioDataset]
+    ) -> Optional[ContainerClass]:
+        if entity.path:
+            return ContainerClass(container=self.get_container_urn(path=entity.path))
+        return None
+
+    def _create_data_platform_instance(self) -> DataPlatformInstanceClass:
+        return DataPlatformInstanceClass(
+            platform=f"urn:li:dataPlatform:{self.platform}",
+            instance=(
+                make_dataplatform_instance_urn(self.platform, self.platform_instance)
+                if self.platform_instance
+                else None
+            ),
+        )
+
+    def _create_dataset_properties(
+        self, dataset: DremioDataset
+    ) -> DatasetPropertiesClass:
+        return DatasetPropertiesClass(
+            name=dataset.resource_name,
+            qualifiedName=f"{'.'.join(dataset.path)}.{dataset.resource_name}",
+            description=dataset.description,
+            externalUrl=self._create_external_url(dataset=dataset),
+            created=TimeStampClass(
+                time=round(
+                    datetime.strptime(
+                        dataset.created, "%Y-%m-%d %H:%M:%S.%f"
+                    ).timestamp()
+                    * 1000
+                )
+                if hasattr(dataset, "created")
+                else 0,
+            ),
+        )
+
+    def _create_external_url(self, dataset: DremioDataset) -> str:
+        container_type = "source"
+        dataset_url_path = '"' + dataset.path[0] + '"/'
+
+        if len(dataset.path) > 1:
+            dataset_url_path = (
+                dataset_url_path + '"' + '"."'.join(dataset.path[1:]) + '".'
+            )
+
+        if dataset.dataset_type == DremioDatasetType.VIEW:
+            container_type = "space"
+        elif dataset.path[0].startswith("@"):
+            container_type = "home"
+
+        return f'{self.ui_url}/{container_type}/{dataset_url_path}"{dataset.resource_name}"'
+
+    def _create_ownership(self, dataset: DremioDataset) -> Optional[OwnershipClass]:
+        if self.ingest_owner and dataset.owner:
+            owner_urn = (
+                make_user_urn(dataset.owner)
+                if dataset.owner_type == "USER"
+                else make_group_urn(dataset.owner)
+            )
+            ownership: OwnershipClass = OwnershipClass(
+                owners=[
+                    OwnerClass(
+                        owner=owner_urn,
+                        type=OwnershipTypeClass.TECHNICAL_OWNER,
+                    )
+                ]
+            )
+            return ownership
+
+        return None
+
+    def _create_glossary_terms(self, entity: DremioDataset) -> GlossaryTermsClass:
+        return GlossaryTermsClass(
+            terms=[
+                GlossaryTermAssociationClass(urn=term.urn)
+                for term in entity.glossary_terms
+            ],
+            auditStamp=AuditStampClass(
+                time=round(time.time() * 1000),
+                actor="urn:li:corpuser:admin",
+            ),
+        )
+
+    def _create_schema_metadata(self, dataset: DremioDataset) -> SchemaMetadataClass:
+        return SchemaMetadataClass(
+            schemaName=f"{'.'.join(dataset.path)}.{dataset.resource_name}",
+            platform=f"urn:li:dataPlatform:{self.platform}",
+            version=0,
+            fields=[self._create_schema_field(column) for column in dataset.columns],
+            platformSchema=MySqlDDLClass(""),
+            hash="",
+        )
+
+    def _create_schema_field(self, column: DremioDatasetColumn) -> SchemaFieldClass:
+        type_class, native_data_type = SchemaFieldTypeMapper.get_field_type(
+            column.data_type,
+            column.column_size,
+        )
+        return SchemaFieldClass(
+            fieldPath=column.name,
+            type=type_class,
+            nativeDataType=native_data_type,
+            nullable=column.is_nullable == "YES",
+        )
+
+    def _create_view_properties(
+        self, dataset: DremioDataset
+    ) -> Optional[ViewPropertiesClass]:
+        if not dataset.sql_definition:
+            return None
+        return ViewPropertiesClass(
+            materialized=False,
+            viewLanguage="SQL",
+            viewLogic=dataset.sql_definition,
+        )
+
+    def _create_glossary_term_info(
+        self, glossary_term: DremioGlossaryTerm
+    ) -> GlossaryTermInfoClass:
+        return GlossaryTermInfoClass(
+            definition="",
+            termSource=self.platform,
+            name=glossary_term.glossary_term,
+        )
+
+    def _create_field_profile(
+        self, field_name: str, field_stats: Dict
+    ) -> DatasetFieldProfileClass:
+        quantiles = field_stats.get("quantiles")
+        return DatasetFieldProfileClass(
+            fieldPath=field_name,
+            uniqueCount=field_stats.get("distinct_count"),
+            nullCount=field_stats.get("null_count"),
+            min=str(field_stats.get("min")) if field_stats.get("min") else None,
+            max=str(field_stats.get("max")) if field_stats.get("max") else None,
+            mean=str(field_stats.get("mean")) if field_stats.get("mean") else None,
+            median=str(field_stats.get("median"))
+            if field_stats.get("median")
+            else None,
+            stdev=str(field_stats.get("stdev")) if field_stats.get("stdev") else None,
+            quantiles=[
+                QuantileClass(quantile=str(0.25), value=str(quantiles[0])),
+                QuantileClass(quantile=str(0.75), value=str(quantiles[1])),
+            ]
+            if quantiles
+            else None,
+            sampleValues=field_stats.get("sample_values"),
+        )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py
new file mode 100644
index 0000000000000..b3f2107a1dfaa
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py
@@ -0,0 +1,181 @@
+import os
+from typing import List, Literal, Optional
+
+import certifi
+from pydantic import Field, validator
+
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.source_common import (
+    EnvConfigMixin,
+    PlatformInstanceConfigMixin,
+)
+from datahub.ingestion.source.ge_profiling_config import GEProfilingBaseConfig
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StatefulStaleMetadataRemovalConfig,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionConfigBase,
+)
+from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
+from datahub.ingestion.source_config.operation_config import is_profiling_enabled
+
+
+class DremioConnectionConfig(ConfigModel):
+    hostname: Optional[str] = Field(
+        default=None,
+        description="Hostname or IP Address of the Dremio server",
+    )
+
+    port: int = Field(
+        default=9047,
+        description="Port of the Dremio REST API",
+    )
+
+    username: Optional[str] = Field(
+        default=None,
+        description="Dremio username",
+    )
+
+    authentication_method: Optional[str] = Field(
+        default="PAT",
+        description="Authentication method: 'password' or 'PAT' (Personal Access Token)",
+    )
+
+    password: Optional[str] = Field(
+        default=None,
+        description="Dremio password or Personal Access Token",
+    )
+
+    tls: bool = Field(
+        default=True,
+        description="Whether the Dremio REST API port is encrypted",
+    )
+
+    disable_certificate_verification: Optional[bool] = Field(
+        default=False,
+        description="Disable TLS certificate verification",
+    )
+
+    path_to_certificates: str = Field(
+        default=certifi.where(),
+        description="Path to SSL certificates",
+    )
+
+    is_dremio_cloud: bool = Field(
+        default=False,
+        description="Whether this is a Dremio Cloud instance",
+    )
+
+    # https://docs.dremio.com/cloud/reference/api/#api-endpoint
+    dremio_cloud_region: Literal["US", "EU"] = Field(
+        default="US",
+        description="Dremio Cloud region ('US' or 'EU')",
+    )
+
+    dremio_cloud_project_id: Optional[str] = Field(
+        default=None,
+        description="ID of Dremio Cloud Project. Found in Project Settings in the Dremio Cloud UI",
+    )
+
+    @validator("authentication_method")
+    def validate_auth_method(cls, value):
+        allowed_methods = ["password", "PAT"]
+        if value not in allowed_methods:
+            raise ValueError(
+                f"authentication_method must be one of {allowed_methods}",
+            )
+        return value
+
+    @validator("password")
+    def validate_password(cls, value, values):
+        if values.get("authentication_method") == "PAT" and not value:
+            raise ValueError(
+                "Password (Personal Access Token) is required when using PAT authentication",
+            )
+        return value
+
+
+class ProfileConfig(GEProfilingBaseConfig):
+    query_timeout: int = Field(
+        default=300, description="Time before cancelling Dremio profiling query"
+    )
+    include_field_median_value: bool = Field(
+        default=False,
+        hidden_from_docs=True,
+        description="Median causes a number of issues in Dremio.",
+    )
+
+
+class DremioSourceMapping(EnvConfigMixin, PlatformInstanceConfigMixin, ConfigModel):
+    platform: str = Field(
+        description="Source connection made by Dremio (e.g. S3, Snowflake)",
+    )
+    source_name: str = Field(
+        description="Alias of platform in Dremio connection",
+    )
+
+
+class DremioSourceConfig(
+    DremioConnectionConfig,
+    StatefulIngestionConfigBase,
+    EnvConfigMixin,
+    PlatformInstanceConfigMixin,
+):
+    domain: Optional[str] = Field(
+        default=None,
+        description="Domain for all source objects.",
+    )
+
+    source_mappings: Optional[List[DremioSourceMapping]] = Field(
+        default=None,
+        description="Mappings from Dremio sources to DataHub platforms and datasets.",
+    )
+
+    # Entity Filters
+    schema_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for schemas to filter",
+    )
+
+    dataset_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for tables and views to filter in ingestion. Specify regex to match the entire table name in dremio.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'dremio.public.customer.*'",
+    )
+
+    usage: BaseUsageConfig = Field(
+        description="The usage config to use when generating usage statistics",
+        default=BaseUsageConfig(),
+    )
+
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
+
+    # Profiling
+    profile_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for tables to profile",
+    )
+    profiling: ProfileConfig = Field(
+        default=ProfileConfig(),
+        description="Configuration for profiling",
+    )
+
+    def is_profiling_enabled(self) -> bool:
+        return self.profiling.enabled and is_profiling_enabled(
+            self.profiling.operation_config
+        )
+
+    # Advanced Configs
+    max_workers: int = Field(
+        default=5 * (os.cpu_count() or 4),
+        description="Number of worker threads to use for parallel processing",
+    )
+
+    include_query_lineage: bool = Field(
+        default=False,
+        description="Whether to include query-based lineage information.",
+    )
+
+    ingest_owner: bool = Field(
+        default=True,
+        description="Ingest Owner from source. This will override Owner info entered from UI",
+    )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py
new file mode 100644
index 0000000000000..e5d6b8e40fb3d
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py
@@ -0,0 +1,116 @@
+from typing import Optional
+
+
+class DremioToDataHubSourceTypeMapping:
+    """
+    Dremio source type to the Datahub source type mapping.
+    """
+
+    SOURCE_TYPE_MAPPING = {
+        # Dremio source types
+        "ADL": "abfs",
+        "AMAZONELASTIC": "elasticsearch",
+        "AWSGLUE": "glue",
+        "AZURE_STORAGE": "abfs",
+        "DB2": "db2",
+        "DREMIOTODREMIO": "dremio",
+        "ELASTIC": "elasticsearch",
+        "GCS": "gcs",
+        "HDFS": "s3",
+        "HIVE": "hive",
+        "HIVE3": "hive",
+        "MONGO": "mongodb",
+        "MSSQL": "mssql",
+        "MYSQL": "mysql",
+        "NAS": "s3",
+        "NESSIE": "iceberg",
+        "ORACLE": "oracle",
+        "POSTGRES": "postgres",
+        "REDSHIFT": "redshift",
+        "S3": "s3",
+        "SNOWFLAKE": "snowflake",
+        "SYNAPSE": "mssql",
+        "TERADATA": "teradata",
+        "VERTICA": "vertica",
+    }
+
+    DATABASE_SOURCE_TYPES = {
+        "AMAZONELASTIC",
+        "AWSGLUE",
+        "AZURE_STORAGE",
+        "DB2",
+        "DREMIOTODREMIO",
+        "ELASTIC",
+        "HIVE",
+        "HIVE3",
+        "MONGO",
+        "MSSQL",
+        "MYSQL",
+        "NESSIE",
+        "ORACLE",
+        "POSTGRES",
+        "REDSHIFT",
+        "SNOWFLAKE",
+        "SYNAPSE",
+        "TERADATA",
+        "VERTICA",
+    }
+
+    FILE_OBJECT_STORAGE_TYPES = {
+        "ADL",
+        "AZURE_STORAGE",
+        "GCS",
+        "HDFS",
+        "NAS",
+        "S3",
+    }
+
+    @staticmethod
+    def get_datahub_source_type(dremio_source_type: str) -> str:
+        """
+        Return the DataHub source type.
+        """
+        return DremioToDataHubSourceTypeMapping.SOURCE_TYPE_MAPPING.get(
+            dremio_source_type.upper(), dremio_source_type.lower()
+        )
+
+    @staticmethod
+    def get_category(source_type: str) -> str:
+        """
+        Define whether the source uses dot notation (DB) or slash notation (Object storage).
+        """
+        if (
+            source_type.upper()
+            in DremioToDataHubSourceTypeMapping.DATABASE_SOURCE_TYPES
+        ):
+            return "database"
+        if (
+            source_type.upper()
+            in DremioToDataHubSourceTypeMapping.FILE_OBJECT_STORAGE_TYPES
+        ):
+            return "file_object_storage"
+        return "unknown"
+
+    @staticmethod
+    def add_mapping(
+        dremio_source_type: str,
+        datahub_source_type: str,
+        category: Optional[str] = None,
+    ) -> None:
+        """
+        Add a new source type if not in the map (e.g., Dremio ARP).
+        """
+        dremio_source_type = dremio_source_type.upper()
+        DremioToDataHubSourceTypeMapping.SOURCE_TYPE_MAPPING[
+            dremio_source_type
+        ] = datahub_source_type
+
+        if category:
+            if category.lower() == "file_object_storage":
+                DremioToDataHubSourceTypeMapping.FILE_OBJECT_STORAGE_TYPES.add(
+                    dremio_source_type
+                )
+            else:
+                DremioToDataHubSourceTypeMapping.DATABASE_SOURCE_TYPES.add(
+                    dremio_source_type
+                )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py
new file mode 100644
index 0000000000000..b80d7b8e0f912
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py
@@ -0,0 +1,462 @@
+import itertools
+import logging
+import re
+import uuid
+from collections import deque
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+from typing import Any, Deque, Dict, List, Optional
+
+from sqlglot import parse_one
+
+from datahub.emitter.mce_builder import make_term_urn
+from datahub.ingestion.source.dremio.dremio_api import (
+    DremioAPIOperations,
+    DremioEdition,
+    DremioEntityContainerType,
+)
+
+logger = logging.getLogger(__name__)
+
+
+QUERY_TYPES = {
+    "DML": ["CREATE", "DELETE", "INSERT", "MERGE", "UPDATE"],
+    "DDL": [
+        "ALTER BRANCH",
+        "ALTER PIPE",
+        "ALTER SOURCE",
+        "ALTER TABLE",
+        "ALTER TAG",
+        "ALTER VIEW",
+        "ANALYZE TABLE",
+        "COPY INTO",
+        "CREATE BRANCH",
+        "CREATE FOLDER",
+        "CREATE PIPE",
+        "CREATE ROLE",
+        "CREATE TABLE",
+        "CREATE TAG",
+        "CREATE USER",
+        "CREATE VIEW",
+        "DESCRIBE PIPE",
+        "DESCRIBE TABLE",
+        "DROP BRANCH",
+        "DROP FOLDER",
+        "DROP PIPE",
+        "DROP ROLE",
+        "DROP TABLE",
+        "DROP TAG",
+        "DROP USER",
+        "DROP VIEW",
+        "GRANT ROLE",
+        "GRANT TO ROLE",
+        "GRANT TO USER",
+        "MERGE BRANCH",
+        "REVOKE FROM ROLE",
+        "REVOKE FROM USER",
+        "REVOKE ROLE",
+        "SHOW BRANCHES",
+        "SHOW CREATE TABLE",
+        "SHOW CREATE VIEW",
+        "SHOW LOGS",
+        "SHOW TABLES",
+        "SHOW TAGS",
+        "SHOW VIEWS",
+        "USE",
+        "VACUUM CATALOG",
+        "VACUUM TABLE",
+    ],
+    "SELECT": ["SELECT", "WITH"],
+    "DATA_MANIPULATION": ["INSERT INTO", "MERGE INTO", "CREATE TABLE"],
+}
+
+
+@dataclass
+class DremioContainerResponse:
+    container_type: str
+    name: str
+    id: str
+    path: Optional[List[str]] = None
+    source_type: Optional[str] = None
+    root_path: Optional[str] = None
+    database_name: Optional[str] = None
+
+
+class DremioDatasetType(Enum):
+    VIEW = "View"
+    TABLE = "Table"
+
+
+class DremioGlossaryTerm:
+    urn: str
+    glossary_term: str
+
+    def __init__(self, glossary_term: str):
+        self.urn = self.set_glossary_term(
+            glossary_term=glossary_term,
+        )
+        self.glossary_term = glossary_term
+
+    def set_glossary_term(self, glossary_term: str) -> str:
+        namespace = uuid.NAMESPACE_DNS
+
+        return make_term_urn(term=str(uuid.uuid5(namespace, glossary_term)))
+
+
+@dataclass
+class DremioDatasetColumn:
+    name: str
+    ordinal_position: int
+    data_type: str
+    column_size: int
+    is_nullable: bool
+
+
+class DremioQuery:
+    query_without_comments: str
+    query_type: str
+    query_subtype: str
+    affected_dataset: str
+
+    def __init__(
+        self,
+        job_id: str,
+        username: str,
+        submitted_ts: str,
+        query: str,
+        queried_datasets: str,
+        affected_datasets: Optional[str] = None,
+    ):
+        self.job_id = job_id
+        self.username = username
+        self.submitted_ts = self._get_submitted_ts(submitted_ts)
+        self.query = self._get_query(query)
+        self.query_without_comments = self.get_raw_query(query)
+        self.query_type = self._get_query_type()
+        self.query_subtype = self._get_query_subtype()
+        self.queried_datasets = self._get_queried_datasets(queried_datasets)
+        if affected_datasets:
+            self.affected_dataset = affected_datasets
+        else:
+            self.affected_dataset = self._get_affected_tables()
+
+    def get(self, attr):
+        return getattr(self, attr, None)
+
+    def _get_submitted_ts(self, timestamp: str) -> datetime:
+        return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")
+
+    def _get_query(self, query: str) -> str:
+        return str(query).replace("'", "'")
+
+    def _get_query_type(self) -> str:
+        query_operator = re.split(
+            pattern=r"\s+",
+            string=self.query_without_comments.strip(),
+            maxsplit=1,
+        )[0]
+
+        if query_operator in QUERY_TYPES["SELECT"]:
+            return "SELECT"
+        if query_operator in QUERY_TYPES["DML"]:
+            return "DML"
+        return "DDL"
+
+    def _get_query_subtype(self) -> str:
+        for query_operator in (
+            QUERY_TYPES["SELECT"] + QUERY_TYPES["DML"] + QUERY_TYPES["DDL"]
+        ):
+            if self.query_without_comments.upper().startswith(query_operator):
+                return query_operator
+        return "UNDEFINED"
+
+    def _get_queried_datasets(self, queried_datasets: str) -> List[str]:
+        return list(
+            {dataset.strip() for dataset in queried_datasets.strip("[]").split(",")}
+        )
+
+    def _get_affected_tables(self) -> str:
+        # TO DO
+        # for manipulation_operator in _data_manipulation_queries:
+        #     if self.query_without_comments.upper().startswith(manipulation_operator):
+
+        return ""
+
+    def get_raw_query(self, sql_query: str) -> str:
+        try:
+            parsed = parse_one(sql_query)
+            return parsed.sql(comments=False)
+        except Exception as e:
+            logger.warning(e)
+            return sql_query
+
+
+class DremioDataset:
+    resource_id: str
+    resource_name: str
+    path: List[str]
+    location_id: str
+    columns: List[DremioDatasetColumn]
+    sql_definition: Optional[str]
+    dataset_type: DremioDatasetType
+    default_schema: Optional[str]
+    owner: Optional[str]
+    owner_type: Optional[str]
+    created: str
+    parents: Optional[List[str]]
+    description: Optional[str]
+    format_type: Optional[str]
+    glossary_terms: List[DremioGlossaryTerm] = []
+
+    def __init__(
+        self,
+        dataset_details: Dict[str, Any],
+        api_operations: DremioAPIOperations,
+    ):
+        self.glossary_terms: List[DremioGlossaryTerm] = []
+        self.resource_id = dataset_details.get("RESOURCE_ID", "")
+        self.resource_name = dataset_details.get("TABLE_NAME", "")
+        self.path = dataset_details.get("TABLE_SCHEMA", "")[1:-1].split(", ")[:-1]
+        self.location_id = dataset_details.get("LOCATION_ID", "")
+        # self.columns = dataset_details.get("COLUMNS", [])
+        # Initialize DremioDatasetColumn instances for each column
+        self.columns = [
+            DremioDatasetColumn(
+                name=col.get("name"),
+                ordinal_position=col.get("ordinal_position"),
+                is_nullable=col.get("is_nullable", False),
+                data_type=col.get("data_type"),
+                column_size=col.get("column_size"),
+            )
+            for col in dataset_details.get("COLUMNS", [])
+        ]
+
+        self.sql_definition = dataset_details.get("VIEW_DEFINITION")
+
+        if self.sql_definition:
+            self.dataset_type = DremioDatasetType.VIEW
+            self.default_schema = api_operations.get_context_for_vds(
+                resource_id=self.resource_id
+            )
+        else:
+            self.dataset_type = DremioDatasetType.TABLE
+
+        self.owner = dataset_details.get("OWNER")
+        self.owner_type = dataset_details.get("OWNER_TYPE")
+
+        if api_operations.edition in (
+            DremioEdition.ENTERPRISE,
+            DremioEdition.CLOUD,
+        ):
+            self.created = dataset_details.get("CREATED", "")
+            self.format_type = dataset_details.get("FORMAT_TYPE")
+
+        self.description = api_operations.get_description_for_resource(
+            resource_id=self.resource_id
+        )
+
+        glossary_terms = api_operations.get_tags_for_resource(
+            resource_id=self.resource_id
+        )
+        if glossary_terms is not None:
+            for glossary_term in glossary_terms:
+                self.glossary_terms.append(
+                    DremioGlossaryTerm(glossary_term=glossary_term)
+                )
+
+        if self.sql_definition and api_operations.edition == DremioEdition.ENTERPRISE:
+            self.parents = api_operations.get_view_parents(
+                dataset_id=self.resource_id,
+            )
+
+
+class DremioContainer:
+    subclass: str = "Dremio Container"
+    container_name: str
+    location_id: str
+    path: List[str]
+    description: Optional[str]
+
+    def __init__(
+        self,
+        container_name: str,
+        location_id: str,
+        path: List[str],
+        api_operations: DremioAPIOperations,
+    ):
+        self.container_name = container_name
+        self.location_id = location_id
+        self.path = path
+
+        self.description = api_operations.get_description_for_resource(
+            resource_id=location_id,
+        )
+
+
+class DremioSource(DremioContainer):
+    subclass: str = "Dremio Source"
+    dremio_source_type: str
+    root_path: Optional[str]
+    database_name: Optional[str]
+
+    def __init__(
+        self,
+        container_name: str,
+        location_id: str,
+        path: List[str],
+        api_operations: DremioAPIOperations,
+        dremio_source_type: str,
+        root_path: Optional[str] = None,
+        database_name: Optional[str] = None,
+    ):
+        super().__init__(
+            container_name=container_name,
+            location_id=location_id,
+            path=path,
+            api_operations=api_operations,
+        )
+        self.dremio_source_type = dremio_source_type
+        self.root_path = root_path
+        self.database_name = database_name
+
+
+class DremioSpace(DremioContainer):
+    subclass: str = "Dremio Space"
+
+
+class DremioFolder(DremioContainer):
+    subclass: str = "Dremio Folder"
+
+
+class DremioCatalog:
+    dremio_api: DremioAPIOperations
+    edition: DremioEdition
+
+    def __init__(self, dremio_api: DremioAPIOperations):
+        self.dremio_api = dremio_api
+        self.edition = dremio_api.edition
+        self.datasets: Deque[DremioDataset] = deque()
+        self.sources: Deque[DremioSource] = deque()
+        self.spaces: Deque[DremioSpace] = deque()
+        self.folders: Deque[DremioFolder] = deque()
+        self.glossary_terms: Deque[DremioGlossaryTerm] = deque()
+        self.queries: Deque[DremioQuery] = deque()
+
+        self.datasets_populated = False
+        self.containers_populated = False
+        self.queries_populated = False
+
+    def set_datasets(self) -> None:
+        if not self.datasets_populated:
+            self.set_containers()
+
+            containers: Deque[DremioContainer] = deque()
+            containers.extend(self.spaces)  # Add DremioSpace elements
+            containers.extend(self.sources)  # Add DremioSource elements
+
+            for dataset_details in self.dremio_api.get_all_tables_and_columns(
+                containers=containers
+            ):
+                dremio_dataset = DremioDataset(
+                    dataset_details=dataset_details,
+                    api_operations=self.dremio_api,
+                )
+                self.datasets.append(dremio_dataset)
+
+                for glossary_term in dremio_dataset.glossary_terms:
+                    if glossary_term not in self.glossary_terms:
+                        self.glossary_terms.append(glossary_term)
+
+            self.datasets_populated = True
+
+    def get_datasets(self) -> Deque[DremioDataset]:
+        self.set_datasets()
+        return self.datasets
+
+    def set_containers(self) -> None:
+        if not self.containers_populated:
+            for container in self.dremio_api.get_all_containers():
+                container_type = container.get("container_type")
+                if container_type == DremioEntityContainerType.SOURCE:
+                    self.sources.append(
+                        DremioSource(
+                            container_name=container.get("name"),
+                            location_id=container.get("id"),
+                            path=[],
+                            api_operations=self.dremio_api,
+                            dremio_source_type=container.get("source_type"),
+                            root_path=container.get("root_path"),
+                            database_name=container.get("database_name"),
+                        )
+                    )
+                elif container_type == DremioEntityContainerType.SPACE:
+                    self.spaces.append(
+                        DremioSpace(
+                            container_name=container.get("name"),
+                            location_id=container.get("id"),
+                            path=[],
+                            api_operations=self.dremio_api,
+                        )
+                    )
+                elif container_type == DremioEntityContainerType.FOLDER:
+                    self.folders.append(
+                        DremioFolder(
+                            container_name=container.get("name"),
+                            location_id=container.get("id"),
+                            path=container.get("path"),
+                            api_operations=self.dremio_api,
+                        )
+                    )
+                else:
+                    self.spaces.append(
+                        DremioSpace(
+                            container_name=container.get("name"),
+                            location_id=container.get("id"),
+                            path=[],
+                            api_operations=self.dremio_api,
+                        )
+                    )
+
+        logging.info("Containers retrieved from source")
+
+        self.containers_populated = True
+
+    def get_containers(self) -> Deque:
+        self.set_containers()
+        return deque(itertools.chain(self.sources, self.spaces, self.folders))
+
+    def get_sources(self) -> Deque[DremioSource]:
+        self.set_containers()
+        return self.sources
+
+    def get_glossary_terms(self) -> Deque[DremioGlossaryTerm]:
+        self.set_datasets()
+        self.set_containers()
+        return self.glossary_terms
+
+    def is_valid_query(self, query: Dict[str, Any]) -> bool:
+        required_fields = [
+            "job_id",
+            "user_name",
+            "submitted_ts",
+            "query",
+            "queried_datasets",
+        ]
+        return all(query.get(field) for field in required_fields)
+
+    def get_queries(self) -> Deque[DremioQuery]:
+        for query in self.dremio_api.extract_all_queries():
+            if not self.is_valid_query(query):
+                continue
+            self.queries.append(
+                DremioQuery(
+                    job_id=query["job_id"],
+                    username=query["user_name"],
+                    submitted_ts=query["submitted_ts"],
+                    query=query["query"],
+                    queried_datasets=query["queried_datasets"],
+                )
+            )
+        self.queries_populated = True
+        return self.queries
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_profiling.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_profiling.py
new file mode 100644
index 0000000000000..5332597ffce9e
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_profiling.py
@@ -0,0 +1,306 @@
+import logging
+import re
+import time
+from typing import Any, Dict, Iterable, List, Tuple
+
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.dremio.dremio_api import (
+    DremioAPIException,
+    DremioAPIOperations,
+)
+from datahub.ingestion.source.dremio.dremio_config import DremioSourceConfig
+from datahub.ingestion.source.dremio.dremio_entities import DremioDataset
+from datahub.ingestion.source.dremio.dremio_reporting import DremioSourceReport
+from datahub.metadata.schema_classes import (
+    DatasetFieldProfileClass,
+    DatasetProfileClass,
+    QuantileClass,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DremioProfiler:
+    config: DremioSourceConfig
+    report: DremioSourceReport
+    MAX_COLUMNS_PER_QUERY = 800
+
+    def __init__(
+        self,
+        config: DremioSourceConfig,
+        report: DremioSourceReport,
+        api_operations: DremioAPIOperations,
+    ) -> None:
+        self.api_operations = api_operations
+        self.config = config
+        self.report = report
+        self.QUERY_TIMEOUT = (
+            config.profiling.query_timeout
+        )  # 5 minutes timeout for each query
+
+    def get_workunits(
+        self, dataset: DremioDataset, dataset_urn: str
+    ) -> Iterable[MetadataWorkUnit]:
+        if not dataset.columns:
+            self.report.warning(
+                message="Skipping profiling as no columns found for table",
+                context=f"{dataset.resource_name}",
+            )
+            self.report.profiling_skipped_other[dataset.resource_name] += 1
+            return
+
+        full_table_name = (
+            '"' + '"."'.join(dataset.path) + '"."' + dataset.resource_name + '"'
+        )
+
+        columns = [(col.name, col.data_type) for col in dataset.columns]
+
+        if not self.config.profile_pattern.allowed(full_table_name):
+            self.report.profiling_skipped_table_profile_pattern[full_table_name] += 1
+            self.report.warning(
+                message="Profiling is restricted due to the specified profile pattern.",
+                context=f"{full_table_name}",
+            )
+            return
+
+        profile_data = self.profile_table(full_table_name, columns)
+        profile_aspect = self.populate_profile_aspect(profile_data)
+
+        if profile_aspect:
+            self.report.report_entity_profiled(dataset.resource_name)
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn, aspect=profile_aspect
+            )
+            yield mcp.as_workunit()
+
+    def populate_profile_aspect(self, profile_data: Dict) -> DatasetProfileClass:
+        field_profiles = [
+            self._create_field_profile(field_name, field_stats)
+            for field_name, field_stats in profile_data.get("column_stats", {}).items()
+        ]
+        return DatasetProfileClass(
+            timestampMillis=round(time.time() * 1000),
+            rowCount=profile_data.get("row_count"),
+            columnCount=profile_data.get("column_count"),
+            fieldProfiles=field_profiles,
+        )
+
+    def _create_field_profile(
+        self, field_name: str, field_stats: Dict
+    ) -> DatasetFieldProfileClass:
+        quantiles = field_stats.get("quantiles")
+        return DatasetFieldProfileClass(
+            fieldPath=field_name,
+            uniqueCount=field_stats.get("distinct_count"),
+            nullCount=field_stats.get("null_count"),
+            min=str(field_stats.get("min")) if field_stats.get("min") else None,
+            max=str(field_stats.get("max")) if field_stats.get("max") else None,
+            mean=str(field_stats.get("mean")) if field_stats.get("mean") else None,
+            median=str(field_stats.get("median"))
+            if field_stats.get("median")
+            else None,
+            stdev=str(field_stats.get("stdev")) if field_stats.get("stdev") else None,
+            quantiles=[
+                QuantileClass(quantile=str(0.25), value=str(quantiles[0])),
+                QuantileClass(quantile=str(0.75), value=str(quantiles[1])),
+            ]
+            if quantiles
+            else None,
+            sampleValues=field_stats.get("sample_values"),
+        )
+
+    def profile_table(self, table_name: str, columns: List[Tuple[str, str]]) -> Dict:
+        chunked_columns = self._chunk_columns(columns)
+        profile_results = []
+
+        for chunk in chunked_columns:
+            try:
+                chunk_result = self._profile_chunk(table_name, chunk)
+                profile_results.append(chunk_result)
+            except DremioAPIException as e:
+                self.report.profiling_skipped_other[table_name] += 1
+                profile_results.append(self._create_empty_profile_result(chunk))
+                self.report.warning(
+                    message="Error in profiling for table",
+                    context=f"{table_name}",
+                    exc=e,
+                )
+        return self._combine_profile_results(profile_results)
+
+    def _profile_chunk(self, table_name: str, columns: List[Tuple[str, str]]) -> Dict:
+        profile_sql = self._build_profile_sql(table_name, columns)
+        try:
+            results = self.api_operations.execute_query(profile_sql)
+            return self._parse_profile_results(results, columns)
+        except DremioAPIException as e:
+            raise e
+
+    def _chunk_columns(
+        self, columns: List[Tuple[str, str]]
+    ) -> List[List[Tuple[str, str]]]:
+        return [
+            columns[i : i + self.MAX_COLUMNS_PER_QUERY]
+            for i in range(0, len(columns), self.MAX_COLUMNS_PER_QUERY)
+        ]
+
+    def _build_profile_sql(
+        self, table_name: str, columns: List[Tuple[str, str]]
+    ) -> str:
+        metrics = []
+
+        metrics.append("COUNT(*) AS row_count")
+        metrics.append(f"{len(columns)} AS column_count")
+
+        if not self.config.profiling.profile_table_level_only:
+            for column_name, data_type in columns:
+                try:
+                    metrics.extend(self._get_column_metrics(column_name, data_type))
+                except Exception as e:
+                    logger.warning(
+                        f"Error building metrics for column {column_name}: {str(e)}"
+                    )
+                    # Skip this column and continue with others
+
+        if not metrics:
+            raise ValueError("No valid metrics could be generated")
+
+        main_query = f"SELECT {', '.join(metrics)} FROM {table_name}"
+
+        if self.config.profiling.limit:
+            main_query += f" LIMIT {self.config.profiling.limit}"
+        if self.config.profiling.offset:
+            main_query += f" OFFSET {self.config.profiling.offset}"
+
+        return main_query
+
+    def _get_column_metrics(self, column_name: str, data_type: str) -> List[str]:
+        metrics = []
+
+        # Wrap column name in quotes to handle special characters
+        quoted_column_name = f'"{column_name}"'
+        safe_column_name = re.sub(r"\W|^(?=\d)", "_", column_name)
+
+        if self.config.profiling.include_field_distinct_count:
+            metrics.append(
+                f"COUNT(DISTINCT {quoted_column_name}) AS {safe_column_name}_distinct_count"
+            )
+
+        if self.config.profiling.include_field_null_count:
+            metrics.append(
+                f"SUM(CASE WHEN {quoted_column_name} IS NULL THEN 1 ELSE 0 END) AS {safe_column_name}_null_count"
+            )
+
+        if self.config.profiling.include_field_min_value:
+            metrics.append(f"MIN({quoted_column_name}) AS {safe_column_name}_min")
+
+        if self.config.profiling.include_field_max_value:
+            metrics.append(f"MAX({quoted_column_name}) AS {safe_column_name}_max")
+
+        if data_type.lower() in [
+            "int",
+            "integer",
+            "bigint",
+            "float",
+            "double",
+            "decimal",
+        ]:
+            if self.config.profiling.include_field_mean_value:
+                metrics.append(
+                    f"AVG(CAST({quoted_column_name} AS DOUBLE)) AS {safe_column_name}_mean"
+                )
+
+            if self.config.profiling.include_field_stddev_value:
+                metrics.append(
+                    f"STDDEV(CAST({quoted_column_name} AS DOUBLE)) AS {safe_column_name}_stdev"
+                )
+
+            if self.config.profiling.include_field_median_value:
+                metrics.append(
+                    f"MEDIAN({quoted_column_name}) AS {safe_column_name}_median"
+                )
+
+            if self.config.profiling.include_field_quantiles:
+                metrics.append(
+                    f"PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY {quoted_column_name}) AS {safe_column_name}_25th_percentile"
+                )
+                metrics.append(
+                    f"PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY {quoted_column_name}) AS {safe_column_name}_75th_percentile"
+                )
+
+        return metrics
+
+    def _parse_profile_results(
+        self, results: List[Dict], columns: List[Tuple[str, str]]
+    ) -> Dict:
+        profile: Dict[str, Any] = {"column_stats": {}}
+        result = results[0] if results else {}  # We expect only one row of results
+
+        profile["row_count"] = int(result.get("row_count", 0))
+
+        profile["column_count"] = int(result.get("column_count", 0))
+
+        for column_name, data_type in columns:
+            safe_column_name = re.sub(r"\W|^(?=\d)", "_", column_name)
+            column_stats: Dict[str, Any] = {}
+            if self.config.profiling.include_field_distinct_count:
+                null_distinct = result.get(f"{safe_column_name}_distinct_count", 0)
+                null_distinct = int(null_distinct) if null_distinct is not None else 0
+                column_stats["distinct_count"] = null_distinct
+
+            if self.config.profiling.include_field_null_count:
+                null_count_value = result.get(f"{safe_column_name}_null_count", 0)
+                null_count = (
+                    int(null_count_value) if null_count_value is not None else 0
+                )
+                column_stats["null_count"] = null_count
+
+            if self.config.profiling.include_field_min_value:
+                column_stats["min"] = result.get(f"{safe_column_name}_min")
+            if self.config.profiling.include_field_max_value:
+                column_stats["max"] = result.get(f"{safe_column_name}_max")
+
+            if data_type.lower() in [
+                "int",
+                "integer",
+                "bigint",
+                "float",
+                "double",
+                "decimal",
+            ]:
+                if self.config.profiling.include_field_mean_value:
+                    column_stats["mean"] = result.get(f"{safe_column_name}_mean")
+                if self.config.profiling.include_field_stddev_value:
+                    column_stats["stdev"] = result.get(f"{safe_column_name}_stdev")
+                if self.config.profiling.include_field_median_value:
+                    column_stats["median"] = result.get(f"{safe_column_name}_median")
+                if self.config.profiling.include_field_quantiles:
+                    column_stats["quantiles"] = [
+                        result.get(f"{safe_column_name}_25th_percentile"),
+                        result.get(f"{safe_column_name}_75th_percentile"),
+                    ]
+
+            profile["column_stats"][column_name] = column_stats
+
+        return profile
+
+    def _create_empty_profile_result(self, columns: List[Tuple[str, str]]) -> Dict:
+        profile: Dict[str, Any] = {"column_stats": {}}
+        for column_name, _ in columns:
+            profile["column_stats"][column_name] = {}
+        return profile
+
+    def _combine_profile_results(self, profile_results: List[Dict]) -> Dict:
+        combined_profile = {}
+        combined_profile["row_count"] = sum(
+            profile.get("row_count", 0) for profile in profile_results
+        )
+        combined_profile["column_count"] = sum(
+            profile.get("column_count", 0) for profile in profile_results
+        )
+        combined_profile["column_stats"] = {}
+
+        for profile in profile_results:
+            combined_profile["column_stats"].update(profile.get("column_stats", {}))
+
+        return combined_profile
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py
new file mode 100644
index 0000000000000..c8eb035461ca1
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py
@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalSourceReport,
+)
+from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
+
+
+@dataclass
+class DremioSourceReport(
+    SQLSourceReport, StaleEntityRemovalSourceReport, IngestionStageReport
+):
+    num_containers_failed: int = 0
+    num_datasets_failed: int = 0
+    containers_scanned: int = 0
+    containers_filtered: int = 0
+
+    def report_upstream_latency(self, start_time: datetime, end_time: datetime) -> None:
+        # recording total combined latency is not very useful, keeping this method as a placeholder
+        # for future implementation of min / max / percentiles etc.
+        pass
+
+    def report_container_scanned(self, name: str) -> None:
+        """
+        Record that a container was successfully scanned
+        """
+        self.containers_scanned += 1
+
+    def report_container_filtered(self, container_name: str) -> None:
+        """
+        Record that a container was filtered out
+        """
+        self.containers_filtered += 1
+        self.report_dropped(container_name)
+
+    def report_entity_scanned(self, name: str, ent_type: str = "View") -> None:
+        """
+        Entity could be a view or a table
+        """
+        if ent_type == "Table":
+            self.tables_scanned += 1
+        elif ent_type == "View":
+            self.views_scanned += 1
+        else:
+            raise KeyError(f"Unknown entity {ent_type}.")
+
+    def set_ingestion_stage(self, dataset: str, stage: str) -> None:
+        self.report_ingestion_stage_start(f"{dataset}: {stage}")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py
new file mode 100644
index 0000000000000..319290d25169a
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py
@@ -0,0 +1,641 @@
+import logging
+import re
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Dict, Iterable, List, Optional
+
+from datahub.emitter.mce_builder import (
+    make_data_platform_urn,
+    make_dataset_urn_with_platform_instance,
+)
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SupportStatus,
+    capability,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import (
+    MetadataWorkUnitProcessor,
+    SourceCapability,
+    SourceReport,
+)
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.dremio.dremio_api import (
+    DremioAPIOperations,
+    DremioEdition,
+)
+from datahub.ingestion.source.dremio.dremio_aspects import DremioAspects
+from datahub.ingestion.source.dremio.dremio_config import DremioSourceConfig
+from datahub.ingestion.source.dremio.dremio_datahub_source_mapping import (
+    DremioToDataHubSourceTypeMapping,
+)
+from datahub.ingestion.source.dremio.dremio_entities import (
+    DremioCatalog,
+    DremioContainer,
+    DremioDataset,
+    DremioDatasetType,
+    DremioGlossaryTerm,
+    DremioQuery,
+)
+from datahub.ingestion.source.dremio.dremio_profiling import DremioProfiler
+from datahub.ingestion.source.dremio.dremio_reporting import DremioSourceReport
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionSourceBase,
+)
+from datahub.ingestion.source_report.ingestion_stage import PROFILING
+from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
+    DatasetLineageTypeClass,
+    UpstreamClass,
+    UpstreamLineage,
+)
+from datahub.metadata.schema_classes import ChangeTypeClass, SchemaMetadataClass
+from datahub.metadata.urns import CorpUserUrn
+from datahub.sql_parsing.sql_parsing_aggregator import (
+    KnownQueryLineageInfo,
+    ObservedQuery,
+    SqlParsingAggregator,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@platform_name("Dremio")
+@config_class(DremioSourceConfig)
+@support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
+@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
+@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
+@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
+@capability(SourceCapability.OWNERSHIP, "Enabled by default")
+@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+class DremioSource(StatefulIngestionSourceBase):
+    """
+    This plugin integrates with Dremio to extract and ingest metadata into DataHub.
+    The following types of metadata are extracted:
+
+    - Metadata for Spaces, Folders, Sources, and Datasets:
+        - Includes physical and virtual datasets, with detailed information about each dataset.
+        - Extracts metadata about Dremio's organizational hierarchy: Spaces (top-level), Folders (sub-level), and Sources (external data connections).
+
+    - Schema and Column Information:
+        - Column types and schema metadata associated with each physical and virtual dataset.
+        - Extracts column-level metadata, such as names, data types, and descriptions, if available.
+
+    - Lineage Information:
+        - Dataset-level and column-level lineage tracking:
+            - Dataset-level lineage shows dependencies and relationships between physical and virtual datasets.
+            - Column-level lineage tracks transformations applied to individual columns across datasets.
+        - Lineage information helps trace the flow of data and transformations within Dremio.
+
+    - Ownership and Glossary Terms:
+        - Metadata related to ownership of datasets, extracted from Dremio’s ownership model.
+        - Glossary terms and business metadata associated with datasets, providing additional context to the data.
+        - Note: Ownership information will only be available for the Cloud and Enterprise editions, it will not be available for the Community edition.
+
+    - Optional SQL Profiling (if enabled):
+        - Table, row, and column statistics can be profiled and ingested via optional SQL queries.
+        - Extracts statistics about tables and columns, such as row counts and data distribution, for better insight into the dataset structure.
+    """
+
+    config: DremioSourceConfig
+    report: DremioSourceReport
+
+    def __init__(self, config: DremioSourceConfig, ctx: PipelineContext):
+        super().__init__(config, ctx)
+        self.default_db = "dremio"
+        self.config = config
+        self.report = DremioSourceReport()
+        self.source_map: Dict[str, Dict] = defaultdict()
+
+        # Initialize API operations
+        dremio_api = DremioAPIOperations(self.config, self.report)
+
+        # Initialize catalog
+        self.dremio_catalog = DremioCatalog(dremio_api)
+
+        # Initialize aspects
+        self.dremio_aspects = DremioAspects(
+            platform=self.get_platform(),
+            domain=self.config.domain,
+            ingest_owner=self.config.ingest_owner,
+            platform_instance=self.config.platform_instance,
+            env=self.config.env,
+            ui_url=dremio_api.ui_url,
+        )
+        self.max_workers = config.max_workers
+
+        self.sql_parsing_aggregator = SqlParsingAggregator(
+            platform=make_data_platform_urn(self.get_platform()),
+            platform_instance=self.config.platform_instance,
+            env=self.config.env,
+            graph=self.ctx.graph,
+            generate_usage_statistics=True,
+            generate_operations=True,
+            usage_config=self.config.usage,
+        )
+
+        # For profiling
+        self.profiler = DremioProfiler(config, self.report, dremio_api)
+
+    @classmethod
+    def create(cls, config_dict: Dict, ctx: PipelineContext) -> "DremioSource":
+        config = DremioSourceConfig.parse_obj(config_dict)
+        return cls(config, ctx)
+
+    def get_platform(self) -> str:
+        return "dremio"
+
+    def _build_source_map(self) -> Dict[str, Dict]:
+        """
+        Builds a source mapping dictionary to support external lineage generation across
+        multiple Dremio sources, based on provided configuration mappings.
+
+        This method operates as follows:
+
+        1. If a source mapping is present in the config:
+        - For each source in the Dremio catalog, if the mapping's `source_name` matches
+            the `dremio_source_type`, `root_path` and `database_name` are added to the mapping
+            information, along with the platform, platform instance, and environment if they exist.
+            This allows constructing the full URN for upstream lineage.
+
+        2. If a source mapping is absent in the configuration:
+        - Default mappings are created for each source name, setting `env` and `platform_instance`
+            to default values and classifying the source type. This ensures all sources have a
+            mapping, even if specific configuration details are missing.
+
+        Returns:
+            Dict[str, Dict]: A dictionary (`source_map`) where each key is a source name
+                            (lowercased) and each value is another dictionary containing:
+                            - `platform`: The source platform.
+                            - `source_name`: The source name.
+                            - `dremio_source_type`: The type mapped to DataHub,
+                            e.g., "database", "folder".
+                            - Optional `root_path`, `database_name`, `platform_instance`,
+                            and `env` if provided in the configuration.
+        Example:
+            This method is used internally within the class to generate mappings before
+            creating cross-platform lineage.
+
+        """
+
+        source_map = {}
+        dremio_sources = self.dremio_catalog.get_sources()
+
+        for source in dremio_sources:
+            source_name = source.container_name
+            if isinstance(source.dremio_source_type, str):
+                source_type = source.dremio_source_type.lower()
+                root_path = source.root_path.lower() if source.root_path else ""
+                database_name = (
+                    source.database_name.lower() if source.database_name else ""
+                )
+                source_present = False
+                source_platform_name = source_name
+
+                for mapping in self.config.source_mappings or []:
+                    if re.search(mapping.source_name, source_type, re.IGNORECASE):
+                        source_platform_name = mapping.source_name.lower()
+
+                    datahub_source_type = (
+                        DremioToDataHubSourceTypeMapping.get_datahub_source_type(
+                            source_type
+                        )
+                    )
+
+                    if re.search(mapping.platform, datahub_source_type, re.IGNORECASE):
+                        source_platform_name = source_platform_name.lower()
+                        source_map[source_platform_name] = {
+                            "platform": mapping.platform,
+                            "source_name": mapping.source_name,
+                            "dremio_source_type": DremioToDataHubSourceTypeMapping.get_category(
+                                source_type,
+                            ),
+                            "root_path": root_path,
+                            "database_name": database_name,
+                            "platform_instance": mapping.platform_instance,
+                            "env": mapping.env,
+                        }
+                        source_present = True
+                        break
+
+                if not source_present:
+                    try:
+                        dremio_source_type = (
+                            DremioToDataHubSourceTypeMapping.get_category(source_type)
+                        )
+                    except Exception as exc:
+                        logger.info(
+                            f"Source {source_type} is not a standard Dremio source type. "
+                            f"Adding source_type {source_type} to mapping as database. Error: {exc}"
+                        )
+
+                        DremioToDataHubSourceTypeMapping.add_mapping(
+                            source_type, source_name
+                        )
+                        dremio_source_type = (
+                            DremioToDataHubSourceTypeMapping.get_category(source_type)
+                        )
+
+                    source_map[source_platform_name.lower()] = {
+                        "platform": source_type,
+                        "source_name": source_name,
+                        "dremio_source_type": dremio_source_type,
+                    }
+
+            else:
+                logger.error(
+                    f'Source "{source.container_name}" is broken. Containers will not be created for source.'
+                )
+                logger.error(
+                    f'No new cross-platform lineage will be emitted for source "{source.container_name}".'
+                )
+                logger.error("Fix this source in Dremio to fix this issue.")
+
+        return source_map
+
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [
+            *super().get_workunit_processors(),
+            StaleEntityRemovalHandler.create(
+                self, self.config, self.ctx
+            ).workunit_processor,
+        ]
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        """
+        Internal method to generate workunits for Dremio metadata.
+        """
+
+        self.source_map = self._build_source_map()
+
+        # Process Containers
+        containers = self.dremio_catalog.get_containers()
+        for container in containers:
+            try:
+                yield from self.process_container(container)
+                logger.info(
+                    f"Dremio container {container.container_name} emitted successfully"
+                )
+            except Exception as exc:
+                self.report.num_containers_failed += 1  # Increment failed containers
+                self.report.report_failure(
+                    message="Failed to process Dremio container",
+                    context=f"{'.'.join(container.path)}.{container.container_name}",
+                    exc=exc,
+                )
+
+        # Process Datasets
+        datasets = self.dremio_catalog.get_datasets()
+
+        for dataset_info in datasets:
+            try:
+                yield from self.process_dataset(dataset_info)
+                logger.info(
+                    f"Dremio dataset {'.'.join(dataset_info.path)}.{dataset_info.resource_name} emitted successfully"
+                )
+            except Exception as exc:
+                self.report.num_datasets_failed += 1  # Increment failed datasets
+                self.report.report_failure(
+                    message="Failed to process Dremio dataset",
+                    context=f"{'.'.join(dataset_info.path)}.{dataset_info.resource_name}",
+                    exc=exc,
+                )
+
+        # Optionally Process Query Lineage
+        if self.config.include_query_lineage:
+            self.get_query_lineage_workunits()
+
+        # Process Glossary Terms
+        glossary_terms = self.dremio_catalog.get_glossary_terms()
+
+        for glossary_term in glossary_terms:
+            try:
+                yield from self.process_glossary_term(glossary_term)
+            except Exception as exc:
+                self.report.report_failure(
+                    message="Failed to process Glossary terms",
+                    context=f"{glossary_term.glossary_term}",
+                    exc=exc,
+                )
+
+        # Generate workunit for aggregated SQL parsing results
+        for mcp in self.sql_parsing_aggregator.gen_metadata():
+            self.report.report_workunit(mcp.as_workunit())
+            yield mcp.as_workunit()
+
+        # Profiling
+        if self.config.is_profiling_enabled():
+            with ThreadPoolExecutor(
+                max_workers=self.config.profiling.max_workers
+            ) as executor:
+                future_to_dataset = {
+                    executor.submit(self.generate_profiles, dataset): dataset
+                    for dataset in datasets
+                }
+
+                for future in as_completed(future_to_dataset):
+                    dataset_info = future_to_dataset[future]
+                    try:
+                        yield from future.result()
+                    except Exception as exc:
+                        self.report.profiling_skipped_other[
+                            dataset_info.resource_name
+                        ] += 1
+                        self.report.report_failure(
+                            message="Failed to profile dataset",
+                            context=f"{'.'.join(dataset_info.path)}.{dataset_info.resource_name}",
+                            exc=exc,
+                        )
+
+    def process_container(
+        self, container_info: DremioContainer
+    ) -> Iterable[MetadataWorkUnit]:
+        """
+        Process a Dremio container and generate metadata workunits.
+        """
+        container_urn = self.dremio_aspects.get_container_urn(
+            path=container_info.path, name=container_info.container_name
+        )
+
+        yield from self.dremio_aspects.populate_container_mcp(
+            container_urn, container_info
+        )
+
+    def process_dataset(
+        self, dataset_info: DremioDataset
+    ) -> Iterable[MetadataWorkUnit]:
+        """
+        Process a Dremio dataset and generate metadata workunits.
+        """
+
+        schema_str = ".".join(dataset_info.path)
+
+        dataset_name = f"{schema_str}.{dataset_info.resource_name}".lower()
+
+        self.report.report_entity_scanned(dataset_name, dataset_info.dataset_type.value)
+        if not self.config.dataset_pattern.allowed(dataset_name):
+            self.report.report_dropped(dataset_name)
+            return
+
+        dataset_urn = make_dataset_urn_with_platform_instance(
+            platform=make_data_platform_urn(self.get_platform()),
+            name=f"dremio.{dataset_name}",
+            env=self.config.env,
+            platform_instance=self.config.platform_instance,
+        )
+
+        for dremio_mcp in self.dremio_aspects.populate_dataset_mcp(
+            dataset_urn, dataset_info
+        ):
+            yield dremio_mcp
+            # Check if the emitted aspect is SchemaMetadataClass
+            if isinstance(
+                dremio_mcp.metadata, MetadataChangeProposalWrapper
+            ) and isinstance(dremio_mcp.metadata.aspect, SchemaMetadataClass):
+                self.sql_parsing_aggregator.register_schema(
+                    urn=dataset_urn,
+                    schema=dremio_mcp.metadata.aspect,
+                )
+
+        if dataset_info.dataset_type == DremioDatasetType.VIEW:
+            if (
+                self.dremio_catalog.edition == DremioEdition.ENTERPRISE
+                and dataset_info.parents
+            ):
+                yield from self.generate_view_lineage(
+                    parents=dataset_info.parents,
+                    dataset_urn=dataset_urn,
+                )
+
+            if dataset_info.sql_definition:
+                self.sql_parsing_aggregator.add_view_definition(
+                    view_urn=dataset_urn,
+                    view_definition=dataset_info.sql_definition,
+                    default_db=self.default_db,
+                    default_schema=dataset_info.default_schema,
+                )
+
+        elif dataset_info.dataset_type == DremioDatasetType.TABLE:
+            dremio_source = dataset_info.path[0] if dataset_info.path else None
+
+            if dremio_source:
+                upstream_urn = self._map_dremio_dataset_to_urn(
+                    dremio_source=dremio_source,
+                    dremio_path=dataset_info.path,
+                    dremio_dataset=dataset_info.resource_name,
+                )
+
+                if upstream_urn:
+                    upstream_lineage = UpstreamLineage(
+                        upstreams=[
+                            UpstreamClass(
+                                dataset=upstream_urn,
+                                type=DatasetLineageTypeClass.COPY,
+                            )
+                        ]
+                    )
+                    mcp = MetadataChangeProposalWrapper(
+                        entityUrn=dataset_urn,
+                        aspect=upstream_lineage,
+                    )
+                    yield mcp.as_workunit()
+                    self.sql_parsing_aggregator.add_known_lineage_mapping(
+                        upstream_urn=upstream_urn,
+                        downstream_urn=dataset_urn,
+                        lineage_type=DatasetLineageTypeClass.COPY,
+                    )
+
+    def process_glossary_term(
+        self, glossary_term_info: DremioGlossaryTerm
+    ) -> Iterable[MetadataWorkUnit]:
+        """
+        Process a Dremio container and generate metadata workunits.
+        """
+
+        yield from self.dremio_aspects.populate_glossary_term_mcp(glossary_term_info)
+
+    def generate_profiles(
+        self, dataset_info: DremioDataset
+    ) -> Iterable[MetadataWorkUnit]:
+        schema_str = ".".join(dataset_info.path)
+        dataset_name = f"{schema_str}.{dataset_info.resource_name}".lower()
+        dataset_urn = make_dataset_urn_with_platform_instance(
+            platform=make_data_platform_urn(self.get_platform()),
+            name=f"dremio.{dataset_name}",
+            env=self.config.env,
+            platform_instance=self.config.platform_instance,
+        )
+        self.report.set_ingestion_stage(dataset_info.resource_name, PROFILING)
+        yield from self.profiler.get_workunits(dataset_info, dataset_urn)
+
+    def generate_view_lineage(
+        self, dataset_urn: str, parents: List[str]
+    ) -> Iterable[MetadataWorkUnit]:
+        """
+        Generate lineage information for views.
+        """
+        upstream_urns = [
+            make_dataset_urn_with_platform_instance(
+                platform=make_data_platform_urn(self.get_platform()),
+                name=f"dremio.{upstream_table.lower()}",
+                env=self.config.env,
+                platform_instance=self.config.platform_instance,
+            )
+            for upstream_table in parents
+        ]
+
+        lineage = UpstreamLineage(
+            upstreams=[
+                UpstreamClass(
+                    dataset=upstream_urn,
+                    type=DatasetLineageTypeClass.VIEW,
+                )
+                for upstream_urn in upstream_urns
+            ]
+        )
+        mcp = MetadataChangeProposalWrapper(
+            entityType="dataset",
+            entityUrn=dataset_urn,
+            aspectName=lineage.ASPECT_NAME,
+            aspect=lineage,
+            changeType=ChangeTypeClass.UPSERT,
+        )
+
+        for upstream_urn in upstream_urns:
+            self.sql_parsing_aggregator.add_known_lineage_mapping(
+                upstream_urn=upstream_urn,
+                downstream_urn=dataset_urn,
+                lineage_type=DatasetLineageTypeClass.VIEW,
+            )
+
+        yield MetadataWorkUnit(id=f"{dataset_urn}-upstreamLineage", mcp=mcp)
+
+    def get_query_lineage_workunits(self) -> None:
+        """
+        Process query lineage information.
+        """
+
+        queries = self.dremio_catalog.get_queries()
+
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            future_to_query = {
+                executor.submit(self.process_query, query): query for query in queries
+            }
+
+            for future in as_completed(future_to_query):
+                query = future_to_query[future]
+                try:
+                    future.result()
+                except Exception as exc:
+                    self.report.report_failure(
+                        message="Failed to process dremio query",
+                        context=f"{query.job_id}: {exc}",
+                        exc=exc,
+                    )
+
+    def process_query(self, query: DremioQuery) -> None:
+        """
+        Process a single Dremio query for lineage information.
+        """
+
+        if query.query and query.affected_dataset:
+            upstream_urns = [
+                make_dataset_urn_with_platform_instance(
+                    platform=make_data_platform_urn(self.get_platform()),
+                    name=f"dremio.{ds.lower()}",
+                    env=self.config.env,
+                    platform_instance=self.config.platform_instance,
+                )
+                for ds in query.queried_datasets
+            ]
+
+            downstream_urn = make_dataset_urn_with_platform_instance(
+                platform=make_data_platform_urn(self.get_platform()),
+                name=f"dremio.{query.affected_dataset.lower()}",
+                env=self.config.env,
+                platform_instance=self.config.platform_instance,
+            )
+
+            # Add query to SqlParsingAggregator
+            self.sql_parsing_aggregator.add_known_query_lineage(
+                KnownQueryLineageInfo(
+                    query_text=query.query,
+                    upstreams=upstream_urns,
+                    downstream=downstream_urn,
+                ),
+                merge_lineage=True,
+            )
+
+        # Add observed query
+        self.sql_parsing_aggregator.add_observed_query(
+            ObservedQuery(
+                query=query.query,
+                timestamp=query.submitted_ts,
+                user=CorpUserUrn(username=query.username),
+                default_db=self.default_db,
+            )
+        )
+
+    def _map_dremio_dataset_to_urn(
+        self,
+        dremio_source: str,
+        dremio_path: List[str],
+        dremio_dataset: str,
+    ) -> Optional[str]:
+        """
+        Map a Dremio dataset to a DataHub URN.
+        """
+        mapping = self.source_map.get(dremio_source.lower())
+        if not mapping:
+            return None
+
+        platform = mapping.get("platform")
+        if not platform:
+            return None
+
+        platform_instance = mapping.get(
+            "platform_instance", self.config.platform_instance
+        )
+        env = mapping.get("env", self.config.env)
+
+        root_path = ""
+        database_name = ""
+
+        if mapping.get("dremio_source_type") == "file_object_storage":
+            if mapping.get("root_path"):
+                root_path = f"{mapping['root_path'][1:]}/"
+            dremio_dataset = f"{root_path}{'/'.join(dremio_path[1:])}/{dremio_dataset}"
+        else:
+            if mapping.get("database_name"):
+                database_name = f"{mapping['database_name']}."
+            dremio_dataset = (
+                f"{database_name}{'.'.join(dremio_path[1:])}.{dremio_dataset}"
+            )
+
+        if platform_instance:
+            return make_dataset_urn_with_platform_instance(
+                platform=platform.lower(),
+                name=dremio_dataset,
+                platform_instance=platform_instance,
+                env=env,
+            )
+
+        return make_dataset_urn_with_platform_instance(
+            platform=platform.lower(),
+            name=dremio_dataset,
+            platform_instance=None,
+            env=env,
+        )
+
+    def get_report(self) -> SourceReport:
+        """
+        Get the source report.
+        """
+        return self.report
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py
new file mode 100644
index 0000000000000..161e8141c8852
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py
@@ -0,0 +1,297 @@
+class DremioSQLQueries:
+    QUERY_DATASETS_CE = """
+    SELECT* FROM
+    (
+    SELECT
+        T.TABLE_SCHEMA,
+        T.TABLE_NAME,
+        CONCAT(T.TABLE_SCHEMA, '.', T.TABLE_NAME) AS FULL_TABLE_PATH,
+        V.VIEW_DEFINITION,
+        C.COLUMN_NAME,
+        C.IS_NULLABLE,
+        C.DATA_TYPE,
+        C.COLUMN_SIZE
+    FROM
+        INFORMATION_SCHEMA."TABLES" T
+        LEFT JOIN INFORMATION_SCHEMA.VIEWS V ON
+        V.TABLE_CATALOG = T.TABLE_CATALOG
+        AND V.TABLE_SCHEMA = T.TABLE_SCHEMA
+        AND V.TABLE_NAME = T.TABLE_NAME
+        INNER JOIN INFORMATION_SCHEMA.COLUMNS C ON
+        C.TABLE_CATALOG = T.TABLE_CATALOG
+        AND C.TABLE_SCHEMA = T.TABLE_SCHEMA
+        AND C.TABLE_NAME = T.TABLE_NAME
+    WHERE
+        T.TABLE_TYPE NOT IN ('SYSTEM_TABLE')
+        AND LOCATE('{container_name}', LOWER(T.TABLE_SCHEMA)) = 1
+    )
+    WHERE 1=1
+        {schema_pattern}
+        {deny_schema_pattern}
+    ORDER BY
+        TABLE_SCHEMA ASC,
+        TABLE_NAME ASC
+    """
+
+    QUERY_DATASETS_EE = """
+        SELECT* FROM
+        (
+        SELECT
+            RESOURCE_ID,
+            V.TABLE_NAME,
+            OWNER,
+            PATH AS TABLE_SCHEMA,
+            CONCAT(REPLACE(REPLACE(
+                REPLACE(V.PATH, ', ', '.'),
+                 '[', ''), ']', ''
+                 )) AS FULL_TABLE_PATH,
+            OWNER_TYPE,
+            LOCATION_ID,
+            VIEW_DEFINITION,
+            FORMAT_TYPE,
+            COLUMN_NAME,
+            ORDINAL_POSITION,
+            IS_NULLABLE,
+            DATA_TYPE,
+            COLUMN_SIZE,
+            CREATED
+        FROM
+            (SELECT
+                VIEW_ID AS RESOURCE_ID,
+                VIEW_NAME AS TABLE_NAME,
+                PATH,
+                CASE
+                    WHEN LENGTH(SCHEMA_ID) = 0 THEN SPACE_ID
+                    ELSE SCHEMA_ID
+                END AS LOCATION_ID,
+                OWNER_ID,
+                SQL_DEFINITION AS VIEW_DEFINITION,
+                '' AS FORMAT_TYPE,
+                CREATED,
+                TYPE
+            FROM
+                SYS.VIEWS
+            UNION ALL
+            SELECT
+                TABLE_ID AS RESOURCE_ID,
+                TABLE_NAME,
+                PATH,
+                CASE
+                    WHEN LENGTH(SCHEMA_ID) = 0 THEN SOURCE_ID
+                    ELSE SCHEMA_ID
+                END AS LOCATION_ID,
+                OWNER_ID,
+                NULL AS VIEW_DEFINITION,
+                FORMAT_TYPE,
+                CREATED,
+                TYPE
+            FROM
+                SYS."TABLES"
+            ) V
+        LEFT JOIN
+            (SELECT
+                USER_ID AS ID,
+                USER_NAME AS "OWNER",
+                'USER' AS OWNER_TYPE
+            FROM
+                SYS.USERS
+            UNION ALL
+            SELECT
+                ROLE_ID AS ID,
+                ROLE_NAME AS "OWNER",
+                'GROUP' AS OWNER_TYPE
+            FROM
+                SYS.ROLES
+            ) U
+        ON
+            V.OWNER_ID = U.ID
+        LEFT JOIN
+        (SELECT
+            TABLE_SCHEMA,
+            TABLE_NAME,
+            COLUMN_NAME,
+            ORDINAL_POSITION,
+            IS_NULLABLE,
+            DATA_TYPE,
+            COLUMN_SIZE
+        FROM
+            INFORMATION_SCHEMA.COLUMNS
+        WHERE
+            LOCATE('{container_name}', LOWER(TABLE_SCHEMA)) = 1
+        ) C
+        ON
+            CONCAT(REPLACE(REPLACE(REPLACE(V.PATH, ', ', '.'), '[', ''), ']', '')) =
+            CONCAT(C.TABLE_SCHEMA, '.', C.TABLE_NAME)
+        WHERE
+            V.TYPE NOT IN ('SYSTEM_TABLE')
+            AND LOCATE('{container_name}', LOWER(PATH)) = 2
+        )
+        WHERE 1=1
+            {schema_pattern}
+            {deny_schema_pattern}
+        ORDER BY
+            TABLE_SCHEMA ASC,
+            TABLE_NAME ASC
+        """
+
+    QUERY_DATASETS_CLOUD = """
+        SELECT * FROM
+        (
+        SELECT
+            RESOURCE_ID,
+            V.TABLE_NAME,
+            OWNER,
+            PATH AS TABLE_SCHEMA,
+            CONCAT(REPLACE(REPLACE(
+            REPLACE(V.PATH, ', ', '.'),
+             '[', ''), ']', ''
+             )) AS FULL_TABLE_PATH,
+            OWNER_TYPE,
+            LOCATION_ID,
+            VIEW_DEFINITION,
+            FORMAT_TYPE,
+            COLUMN_NAME,
+            ORDINAL_POSITION,
+            IS_NULLABLE,
+            DATA_TYPE,
+            COLUMN_SIZE,
+            CREATED
+        FROM
+            (SELECT
+                VIEW_ID AS RESOURCE_ID,
+                VIEW_NAME AS TABLE_NAME,
+                PATH,
+                CASE
+                    WHEN LENGTH(SCHEMA_ID) = 0 THEN SPACE_ID
+                    ELSE SCHEMA_ID
+                END AS LOCATION_ID,
+                OWNER_ID,
+                SQL_DEFINITION AS VIEW_DEFINITION,
+                '' AS FORMAT_TYPE,
+                CREATED,
+                TYPE
+            FROM
+                SYS.PROJECT.VIEWS
+            UNION ALL
+            SELECT
+                TABLE_ID AS RESOURCE_ID,
+                TABLE_NAME,
+                PATH,
+                CASE
+                    WHEN LENGTH(SCHEMA_ID) = 0 THEN SOURCE_ID
+                    ELSE SCHEMA_ID
+                END AS LOCATION_ID,
+                OWNER_ID,
+                NULL AS VIEW_DEFINITION,
+                FORMAT_TYPE,
+                CREATED,
+                TYPE
+            FROM
+                SYS.PROJECT."TABLES"
+            ) V
+        LEFT JOIN
+            (SELECT
+                USER_ID AS ID,
+                USER_NAME AS "OWNER",
+                'USER' AS OWNER_TYPE
+            FROM
+                SYS.ORGANIZATION.USERS
+            UNION ALL
+            SELECT
+                ROLE_ID AS ID,
+                ROLE_NAME AS "OWNER",
+                'GROUP' AS OWNER_TYPE
+            FROM
+                SYS.ORGANIZATION.ROLES
+            ) U
+        ON
+            V.OWNER_ID = U.ID
+        LEFT JOIN
+        (SELECT
+            TABLE_SCHEMA,
+            TABLE_NAME,
+            COLUMN_NAME,
+            ORDINAL_POSITION,
+            IS_NULLABLE,
+            DATA_TYPE,
+            COLUMN_SIZE
+        FROM
+            INFORMATION_SCHEMA.COLUMNS
+        WHERE
+            LOCATE('{container_name}', LOWER(TABLE_SCHEMA)) = 1
+        ) C
+        ON
+            CONCAT(REPLACE(REPLACE(REPLACE(V.PATH, ', ', '.'), '[', ''), ']', '')) =
+            CONCAT(C.TABLE_SCHEMA, '.', C.TABLE_NAME)
+        WHERE
+            V.TYPE NOT IN ('SYSTEM_TABLE')
+            AND LOCATE('{container_name}', LOWER(PATH)) = 2
+        )
+        WHERE 1=1
+            {schema_pattern}
+            {deny_schema_pattern}
+        ORDER BY
+            TABLE_SCHEMA ASC,
+            TABLE_NAME ASC
+            """
+
+    # Dremio Documentation: https://docs.dremio.com/current/reference/sql/system-tables/jobs_recent/
+    # queried_datasets incorrectly documented as [varchar]. Observed as varchar.
+    # LENGTH used as opposed to ARRAY_SIZE
+    QUERY_ALL_JOBS = """
+    SELECT
+        job_id,
+        user_name,
+        submitted_ts,
+        query,
+        queried_datasets
+    FROM
+        SYS.JOBS_RECENT
+    WHERE
+        STATUS = 'COMPLETED'
+        AND LENGTH(queried_datasets)>0
+        AND user_name != '$dremio$'
+        AND query_type not like '%INTERNAL%'
+    """
+
+    # Dremio Documentation: https://docs.dremio.com/cloud/reference/sql/system-tables/jobs-historical
+    # queried_datasets correctly documented as [varchar]
+    QUERY_ALL_JOBS_CLOUD = """
+        SELECT
+            job_id,
+            user_name,
+            submitted_ts,
+            query,
+            CONCAT('[', ARRAY_TO_STRING(queried_datasets, ','), ']') as queried_datasets
+        FROM
+            sys.project.history.jobs
+        WHERE
+            STATUS = 'COMPLETED'
+            AND ARRAY_SIZE(queried_datasets)>0
+            AND user_name != '$dremio$'
+            AND query_type not like '%INTERNAL%'
+        """
+
+    QUERY_TYPES = [
+        "ALTER TABLE",
+        "ALTER VIEW",
+        "COPY INTO",
+        "CREATE TABLE",
+        "CREATE VIEW",
+        "DROP TABLE",
+        "DROP VIEW",
+        "SELECT",
+        "WITH",
+    ]
+
+    PROFILE_COLUMNS = """
+    SELECT
+        {profile_queries}
+    FROM(
+    SELECT
+        *
+    FROM
+        {dremio_dataset}
+    LIMIT {sample_limit}
+    )
+    """
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py
index acda656526ef5..cb3f0dd9cf29f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py
@@ -52,24 +52,22 @@
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionSourceBase,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     BooleanTypeClass,
     BytesTypeClass,
+    DataPlatformInstanceClass,
+    DatasetPropertiesClass,
     NullTypeClass,
     NumberTypeClass,
     RecordTypeClass,
-    SchemaField,
-    SchemaFieldDataType,
+    SchemaFieldClass as SchemaField,
+    SchemaFieldDataTypeClass as SchemaFieldDataType,
     SchemalessClass,
-    SchemaMetadata,
+    SchemaMetadataClass,
     StringTypeClass,
     UnionTypeClass,
 )
-from datahub.metadata.schema_classes import (
-    DataPlatformInstanceClass,
-    DatasetPropertiesClass,
-)
 from datahub.utilities.registries.domain_registry import DomainRegistry
 
 MAX_ITEMS_TO_RETRIEVE = 100
@@ -235,7 +233,6 @@ def _process_table(
         table_name: str,
         dataset_name: str,
     ) -> Iterable[MetadataWorkUnit]:
-
         logger.debug(f"Processing table: {dataset_name}")
         table_info = dynamodb_client.describe_table(TableName=table_name)["Table"]
         account_id = table_info["TableArn"].split(":")[4]
@@ -448,7 +445,7 @@ def construct_schema_metadata(
         dataset_properties: DatasetPropertiesClass,
         schema: Dict[Tuple[str, ...], SchemaDescription],
         primary_key_dict: Dict[str, str],
-    ) -> SchemaMetadata:
+    ) -> SchemaMetadataClass:
         """ "
         To construct the schema metadata, it will first sort the schema by the occurrence of attribute names
         in descending order and truncate the schema by MAX_SCHEMA_SIZE, and then start to construct the
@@ -502,7 +499,7 @@ def construct_schema_metadata(
             canonical_schema.append(field)
 
         # create schema metadata object for table
-        schema_metadata = SchemaMetadata(
+        schema_metadata = SchemaMetadataClass(
             schemaName=table_name,
             platform=f"urn:li:dataPlatform:{self.platform}",
             version=0,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py
index aa5913f5dc66b..99aa5f54f6a57 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py
@@ -227,7 +227,7 @@ def collapse_name(name: str, collapse_urns: CollapseUrns) -> str:
 def collapse_urn(urn: str, collapse_urns: CollapseUrns) -> str:
     if len(collapse_urns.urns_suffix_regex) == 0:
         return urn
-    urn_obj = DatasetUrn.create_from_string(urn)
+    urn_obj = DatasetUrn.from_string(urn)
     name = collapse_name(name=urn_obj.get_dataset_name(), collapse_urns=collapse_urns)
     data_platform_urn = urn_obj.get_data_platform_urn()
     return str(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/feast.py b/metadata-ingestion/src/datahub/ingestion/source/feast.py
index e097fd1f221ea..6330fe0291660 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/feast.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/feast.py
@@ -42,10 +42,14 @@
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.schema_classes import (
     BrowsePathsClass,
+    GlobalTagsClass,
     MLFeaturePropertiesClass,
     MLFeatureTablePropertiesClass,
     MLPrimaryKeyPropertiesClass,
+    OwnerClass,
+    OwnershipClass,
     StatusClass,
+    TagAssociationClass,
 )
 
 # FIXME: ValueType module cannot be used as a type
@@ -91,6 +95,24 @@ class FeastRepositorySourceConfig(ConfigModel):
     environment: str = Field(
         default=DEFAULT_ENV, description="Environment to use when constructing URNs"
     )
+    # owner_mappings example:
+    # This must be added to the recipe in order to extract owners, otherwise NO owners will be extracted
+    # owner_mappings:
+    #   - feast_owner_name: "<owner>"
+    #     datahub_owner_urn: "urn:li:corpGroup:<owner>"
+    #     datahub_ownership_type: "BUSINESS_OWNER"
+    owner_mappings: Optional[List[Dict[str, str]]] = Field(
+        default=None, description="Mapping of owner names to owner types"
+    )
+    enable_owner_extraction: bool = Field(
+        default=False,
+        description="If this is disabled, then we NEVER try to map owners. "
+        "If this is enabled, then owner_mappings is REQUIRED to extract ownership.",
+    )
+    enable_tag_extraction: bool = Field(
+        default=False,
+        description="If this is disabled, then we NEVER try to extract tags.",
+    )
 
 
 @platform_name("Feast")
@@ -215,10 +237,15 @@ def _get_entity_workunit(
         """
 
         feature_view_name = f"{self.feature_store.project}.{feature_view.name}"
+        aspects = (
+            [StatusClass(removed=False)]
+            + self._get_tags(entity)
+            + self._get_owners(entity)
+        )
 
         entity_snapshot = MLPrimaryKeySnapshot(
             urn=builder.make_ml_primary_key_urn(feature_view_name, entity.name),
-            aspects=[StatusClass(removed=False)],
+            aspects=aspects,
         )
 
         entity_snapshot.aspects.append(
@@ -243,10 +270,11 @@ def _get_feature_workunit(
         Generate an MLFeature work unit for a Feast feature.
         """
         feature_view_name = f"{self.feature_store.project}.{feature_view.name}"
+        aspects = [StatusClass(removed=False)] + self._get_tags(field)
 
         feature_snapshot = MLFeatureSnapshot(
             urn=builder.make_ml_feature_urn(feature_view_name, field.name),
-            aspects=[StatusClass(removed=False)],
+            aspects=aspects,
         )
 
         feature_sources = []
@@ -295,13 +323,18 @@ def _get_feature_view_workunit(self, feature_view: FeatureView) -> MetadataWorkU
         """
 
         feature_view_name = f"{self.feature_store.project}.{feature_view.name}"
+        aspects = (
+            [
+                BrowsePathsClass(paths=[f"/feast/{self.feature_store.project}"]),
+                StatusClass(removed=False),
+            ]
+            + self._get_tags(feature_view)
+            + self._get_owners(feature_view)
+        )
 
         feature_view_snapshot = MLFeatureTableSnapshot(
             urn=builder.make_ml_feature_table_urn("feast", feature_view_name),
-            aspects=[
-                BrowsePathsClass(paths=[f"/feast/{self.feature_store.project}"]),
-                StatusClass(removed=False),
-            ],
+            aspects=aspects,
         )
 
         feature_view_snapshot.aspects.append(
@@ -360,6 +393,64 @@ def _get_on_demand_feature_view_workunit(
 
         return MetadataWorkUnit(id=on_demand_feature_view_name, mce=mce)
 
+    # If a tag is specified in a Feast object, then the tag will be ingested into Datahub if enable_tag_extraction is
+    # True, otherwise NO tags will be ingested
+    def _get_tags(self, obj: Union[Entity, FeatureView, FeastField]) -> list:
+        """
+        Extracts tags from the given object and returns a list of aspects.
+        """
+        aspects: List[Union[GlobalTagsClass]] = []
+
+        # Extract tags
+        if self.source_config.enable_tag_extraction:
+            if obj.tags.get("name"):
+                tag_name: str = obj.tags["name"]
+                tag_association = TagAssociationClass(
+                    tag=builder.make_tag_urn(tag_name)
+                )
+                global_tags_aspect = GlobalTagsClass(tags=[tag_association])
+                aspects.append(global_tags_aspect)
+
+        return aspects
+
+    # If an owner is specified in a Feast object, it will only be ingested into Datahub if owner_mappings is specified
+    # and enable_owner_extraction is True in FeastRepositorySourceConfig, otherwise NO owners will be ingested
+    def _get_owners(self, obj: Union[Entity, FeatureView, FeastField]) -> list:
+        """
+        Extracts owners from the given object and returns a list of aspects.
+        """
+        aspects: List[Union[OwnershipClass]] = []
+
+        # Extract owner
+        if self.source_config.enable_owner_extraction:
+            owner = getattr(obj, "owner", None)
+            if owner:
+                # Create owner association, skipping if None
+                owner_association = self._create_owner_association(owner)
+                if owner_association:  # Only add valid owner associations
+                    owners_aspect = OwnershipClass(owners=[owner_association])
+                    aspects.append(owners_aspect)
+
+        return aspects
+
+    def _create_owner_association(self, owner: str) -> Optional[OwnerClass]:
+        """
+        Create an OwnerClass instance for the given owner using the owner mappings.
+        """
+        if self.source_config.owner_mappings is not None:
+            for mapping in self.source_config.owner_mappings:
+                if mapping["feast_owner_name"] == owner:
+                    ownership_type_class: str = mapping.get(
+                        "datahub_ownership_type", "TECHNICAL_OWNER"
+                    )
+                    datahub_owner_urn = mapping.get("datahub_owner_urn")
+                    if datahub_owner_urn:
+                        return OwnerClass(
+                            owner=datahub_owner_urn,
+                            type=ownership_type_class,
+                        )
+        return None
+
     @classmethod
     def create(cls, config_dict, ctx):
         config = FeastRepositorySourceConfig.parse_obj(config_dict)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/file.py b/metadata-ingestion/src/datahub/ingestion/source/file.py
index 853487b1f1c9f..0b1665d579191 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/file.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/file.py
@@ -12,7 +12,7 @@
 from pydantic import validator
 from pydantic.fields import Field
 
-from datahub.configuration.common import ConfigEnum, ConfigModel
+from datahub.configuration.common import ConfigEnum
 from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -26,14 +26,25 @@
 from datahub.ingestion.api.source import (
     CapabilityReport,
     MetadataWorkUnitProcessor,
-    SourceReport,
     TestableSource,
     TestConnectionReport,
 )
-from datahub.ingestion.api.source_helpers import auto_workunit_reporter
+from datahub.ingestion.api.source_helpers import (
+    auto_status_aspect,
+    auto_workunit_reporter,
+)
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.fs.fs_base import FileInfo, get_path_schema
 from datahub.ingestion.fs.fs_registry import fs_registry
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+    StaleEntityRemovalSourceReport,
+    StatefulStaleMetadataRemovalConfig,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionConfigBase,
+    StatefulIngestionSourceBase,
+)
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import (
     MetadataChangeEvent,
     MetadataChangeProposal,
@@ -49,7 +60,7 @@ class FileReadMode(ConfigEnum):
     AUTO = auto()
 
 
-class FileSourceConfig(ConfigModel):
+class FileSourceConfig(StatefulIngestionConfigBase):
     _filename = pydantic_field_deprecated(
         "filename",
         message="filename is deprecated. Use path instead.",
@@ -88,6 +99,8 @@ class FileSourceConfig(ConfigModel):
         "filename", "path", print_warning=False
     )
 
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
+
     @validator("file_extension", always=True)
     def add_leading_dot_to_extension(cls, v: str) -> str:
         if v:
@@ -99,7 +112,7 @@ def add_leading_dot_to_extension(cls, v: str) -> str:
 
 
 @dataclass
-class FileSourceReport(SourceReport):
+class FileSourceReport(StaleEntityRemovalSourceReport):
     total_num_files: int = 0
     num_files_completed: int = 0
     files_completed: list = field(default_factory=list)
@@ -174,7 +187,7 @@ def compute_stats(self) -> None:
 @platform_name("Metadata File")
 @config_class(FileSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
-class GenericFileSource(TestableSource):
+class GenericFileSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin pulls metadata from a previously generated file.
     The [metadata file sink](../../../../metadata-ingestion/sink_docs/metadata-file.md) can produce such files, and a number of
@@ -182,9 +195,10 @@ class GenericFileSource(TestableSource):
     """
 
     def __init__(self, ctx: PipelineContext, config: FileSourceConfig):
+        super().__init__(config, ctx)
         self.ctx = ctx
         self.config = config
-        self.report = FileSourceReport()
+        self.report: FileSourceReport = FileSourceReport()
 
     @classmethod
     def create(cls, config_dict, ctx):
@@ -204,7 +218,13 @@ def get_filenames(self) -> Iterable[FileInfo]:
 
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         # No super() call, as we don't want helpers that create / remove workunits
-        return [partial(auto_workunit_reporter, self.report)]
+        return [
+            partial(auto_workunit_reporter, self.report),
+            auto_status_aspect if self.config.stateful_ingestion else None,
+            StaleEntityRemovalHandler.create(
+                self, self.config, self.ctx
+            ).workunit_processor,
+        ]
 
     def get_workunits_internal(
         self,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py
index 02eb096b240f5..86826ae7bedc0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py
@@ -1,14 +1,20 @@
+import dataclasses
 import logging
-from dataclasses import dataclass, field as dataclass_field
-from typing import Dict, List, Optional
+import warnings
+from typing import Dict, Optional
 
 import pydantic
 from pydantic import Field, root_validator
 from typing_extensions import Literal
 
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
-from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin
+from datahub.configuration.common import (
+    AllowDenyPattern,
+    ConfigModel,
+    ConfigurationWarning,
+)
+from datahub.configuration.source_common import DatasetSourceConfigMixin
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
+from datahub.emitter.mce_builder import DEFAULT_ENV
 from datahub.ingestion.api.report import Report
 from datahub.ingestion.source.bigquery_v2.bigquery_config import (
     BigQueryConnectionConfig,
@@ -23,6 +29,7 @@
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
 )
+from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.perf_timer import PerfTimer
 
 logger = logging.getLogger(__name__)
@@ -114,35 +121,39 @@ def validate_destination_platfrom_and_config(cls, values: Dict) -> Dict:
         return values
 
 
-@dataclass
+@dataclasses.dataclass
 class MetadataExtractionPerfReport(Report):
-    connectors_metadata_extraction_sec: PerfTimer = dataclass_field(
+    connectors_metadata_extraction_sec: PerfTimer = dataclasses.field(
         default_factory=PerfTimer
     )
-    connectors_lineage_extraction_sec: PerfTimer = dataclass_field(
+    connectors_lineage_extraction_sec: PerfTimer = dataclasses.field(
         default_factory=PerfTimer
     )
-    connectors_jobs_extraction_sec: PerfTimer = dataclass_field(
+    connectors_jobs_extraction_sec: PerfTimer = dataclasses.field(
         default_factory=PerfTimer
     )
 
 
-@dataclass
+@dataclasses.dataclass
 class FivetranSourceReport(StaleEntityRemovalSourceReport):
     connectors_scanned: int = 0
-    filtered_connectors: List[str] = dataclass_field(default_factory=list)
-    metadata_extraction_perf: MetadataExtractionPerfReport = dataclass_field(
+    filtered_connectors: LossyList[str] = dataclasses.field(default_factory=LossyList)
+    metadata_extraction_perf: MetadataExtractionPerfReport = dataclasses.field(
         default_factory=MetadataExtractionPerfReport
     )
 
     def report_connectors_scanned(self, count: int = 1) -> None:
         self.connectors_scanned += count
 
-    def report_connectors_dropped(self, model: str) -> None:
-        self.filtered_connectors.append(model)
+    def report_connectors_dropped(self, connector: str) -> None:
+        self.filtered_connectors.append(connector)
 
 
 class PlatformDetail(ConfigModel):
+    platform: Optional[str] = pydantic.Field(
+        default=None,
+        description="Override the platform type detection.",
+    )
     platform_instance: Optional[str] = pydantic.Field(
         default=None,
         description="The instance of the platform that all assets produced by this recipe belong to",
@@ -151,6 +162,11 @@ class PlatformDetail(ConfigModel):
         default=DEFAULT_ENV,
         description="The environment that all assets produced by DataHub platform ingestion source belong to",
     )
+    database: Optional[str] = pydantic.Field(
+        default=None,
+        description="The database that all assets produced by this connector belong to. "
+        "For destinations, this defaults to the fivetran log config's database.",
+    )
 
 
 class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin):
@@ -159,34 +175,54 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin
     )
     connector_patterns: AllowDenyPattern = Field(
         default=AllowDenyPattern.allow_all(),
-        description="Regex patterns for connectors to filter in ingestion.",
+        description="Filtering regex patterns for connector names.",
     )
     destination_patterns: AllowDenyPattern = Field(
         default=AllowDenyPattern.allow_all(),
-        description="Regex patterns for destinations to filter in ingestion.",
+        description="Regex patterns for destination ids to filter in ingestion. "
+        "Fivetran destination IDs are usually two word identifiers e.g. canyon_tolerable, and are not the same as the destination database name. "
+        "They're visible in the Fivetran UI under Destinations -> Overview -> Destination Group ID.",
     )
     include_column_lineage: bool = Field(
         default=True,
         description="Populates table->table column lineage.",
     )
-    sources_to_database: Dict[str, str] = pydantic.Field(
-        default={},
-        description="A mapping of the connector's all sources to its database. Use connector id as key.",
-    )
+
     # Configuration for stateful ingestion
     stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field(
         default=None, description="Airbyte Stateful Ingestion Config."
     )
+
     # Fivetran connector all sources to platform instance mapping
     sources_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field(
         default={},
-        description="A mapping of the connector's all sources dataset to platform instance. Use connector id as key.",
+        description="A mapping from connector id to its platform/instance/env/database details.",
     )
     # Fivetran destination to platform instance mapping
     destination_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field(
         default={},
-        description="A mapping of destination dataset to platform instance. Use destination id as key.",
+        description="A mapping of destination id to its platform/instance/env details.",
     )
+
+    @pydantic.root_validator(pre=True)
+    def compat_sources_to_database(cls, values: Dict) -> Dict:
+        if "sources_to_database" in values:
+            warnings.warn(
+                "The sources_to_database field is deprecated, please use sources_to_platform_instance instead.",
+                ConfigurationWarning,
+                stacklevel=2,
+            )
+            mapping = values.pop("sources_to_database")
+
+            values.setdefault("sources_to_platform_instance", {})
+            for key, value in mapping.items():
+                values["sources_to_platform_instance"].setdefault(key, {})
+                values["sources_to_platform_instance"][key].setdefault(
+                    "database", value
+                )
+
+        return values
+
     history_sync_lookback_period: int = pydantic.Field(
         7,
         description="The number of days to look back when extracting connectors' sync history.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py
index 18de2b01edd3b..046aa9efe3f59 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py
@@ -24,7 +24,7 @@ class Connector:
     sync_frequency: int
     destination_id: str
     user_id: str
-    table_lineage: List[TableLineage]
+    lineage: List[TableLineage]
     jobs: List["Job"]
 
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
index 704a6f20a5c19..adbfc48692db9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
@@ -7,7 +7,6 @@
     DataProcessInstance,
     InstanceRunResult,
 )
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SourceCapability,
@@ -28,6 +27,10 @@
 )
 from datahub.ingestion.source.fivetran.data_classes import Connector, Job
 from datahub.ingestion.source.fivetran.fivetran_log_api import FivetranLogAPI
+from datahub.ingestion.source.fivetran.fivetran_query import (
+    MAX_JOBS_PER_CONNECTOR,
+    MAX_TABLE_LINEAGE_PER_CONNECTOR,
+)
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
 )
@@ -39,7 +42,6 @@
     FineGrainedLineageDownstreamType,
     FineGrainedLineageUpstreamType,
 )
-from datahub.metadata.schema_classes import StatusClass
 from datahub.utilities.urns.data_flow_urn import DataFlowUrn
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 
@@ -72,65 +74,73 @@ def __init__(self, config: FivetranSourceConfig, ctx: PipelineContext):
 
         self.audit_log = FivetranLogAPI(self.config.fivetran_log_config)
 
-        # Create and register the stateful ingestion use-case handler.
-        self.stale_entity_removal_handler = StaleEntityRemovalHandler.create(
-            self, self.config, self.ctx
-        )
-
-    def _extend_lineage(self, connector: Connector, datajob: DataJob) -> None:
+    def _extend_lineage(self, connector: Connector, datajob: DataJob) -> Dict[str, str]:
         input_dataset_urn_list: List[DatasetUrn] = []
         output_dataset_urn_list: List[DatasetUrn] = []
         fine_grained_lineage: List[FineGrainedLineage] = []
 
-        source_platform_detail: PlatformDetail = PlatformDetail()
-        destination_platform_detail: PlatformDetail = PlatformDetail()
+        # TODO: Once Fivetran exposes the database via the API, we shouldn't ask for it via config.
+
         # Get platform details for connector source
-        source_platform_detail = self.config.sources_to_platform_instance.get(
+        source_details = self.config.sources_to_platform_instance.get(
             connector.connector_id, PlatformDetail()
         )
+        if source_details.platform is None:
+            if connector.connector_type in KNOWN_DATA_PLATFORM_MAPPING:
+                source_details.platform = KNOWN_DATA_PLATFORM_MAPPING[
+                    connector.connector_type
+                ]
+            else:
+                self.report.info(
+                    title="Guessing source platform for lineage",
+                    message="We encountered a connector type that we don't fully support yet. "
+                    "We will attempt to guess the platform based on the connector type.",
+                    context=f"{connector.connector_name} (connector_id: {connector.connector_id}, connector_type: {connector.connector_type})",
+                )
+                source_details.platform = connector.connector_type
 
         # Get platform details for destination
-        destination_platform_detail = self.config.destination_to_platform_instance.get(
+        destination_details = self.config.destination_to_platform_instance.get(
             connector.destination_id, PlatformDetail()
         )
-
-        # Get database for connector source
-        # TODO: Once Fivetran exposes this, we shouldn't ask for it via config.
-        source_database: Optional[str] = self.config.sources_to_database.get(
-            connector.connector_id
-        )
-
-        if connector.connector_type in KNOWN_DATA_PLATFORM_MAPPING:
-            source_platform = KNOWN_DATA_PLATFORM_MAPPING[connector.connector_type]
-        else:
-            source_platform = connector.connector_type
-            logger.info(
-                f"Fivetran connector source type: {connector.connector_type} is not supported to mapped with Datahub dataset entity."
+        if destination_details.platform is None:
+            destination_details.platform = (
+                self.config.fivetran_log_config.destination_platform
+            )
+        if destination_details.database is None:
+            destination_details.database = self.audit_log.fivetran_log_database
+
+        if len(connector.lineage) >= MAX_TABLE_LINEAGE_PER_CONNECTOR:
+            self.report.warning(
+                title="Table lineage truncated",
+                message=f"The connector had more than {MAX_TABLE_LINEAGE_PER_CONNECTOR} table lineage entries. "
+                f"Only the most recent {MAX_TABLE_LINEAGE_PER_CONNECTOR} entries were ingested.",
+                context=f"{connector.connector_name} (connector_id: {connector.connector_id})",
             )
 
-        for table_lineage in connector.table_lineage:
+        for lineage in connector.lineage:
             input_dataset_urn = DatasetUrn.create_from_ids(
-                platform_id=source_platform,
+                platform_id=source_details.platform,
                 table_name=(
-                    f"{source_database.lower()}.{table_lineage.source_table}"
-                    if source_database
-                    else table_lineage.source_table
+                    f"{source_details.database.lower()}.{lineage.source_table}"
+                    if source_details.database
+                    else lineage.source_table
                 ),
-                env=source_platform_detail.env,
-                platform_instance=source_platform_detail.platform_instance,
+                env=source_details.env,
+                platform_instance=source_details.platform_instance,
             )
             input_dataset_urn_list.append(input_dataset_urn)
 
             output_dataset_urn = DatasetUrn.create_from_ids(
-                platform_id=self.config.fivetran_log_config.destination_platform,
-                table_name=f"{self.audit_log.fivetran_log_database.lower()}.{table_lineage.destination_table}",
-                env=destination_platform_detail.env,
-                platform_instance=destination_platform_detail.platform_instance,
+                platform_id=destination_details.platform,
+                table_name=f"{destination_details.database.lower()}.{lineage.destination_table}",
+                env=destination_details.env,
+                platform_instance=destination_details.platform_instance,
             )
             output_dataset_urn_list.append(output_dataset_urn)
 
             if self.config.include_column_lineage:
-                for column_lineage in table_lineage.column_lineage:
+                for column_lineage in lineage.column_lineage:
                     fine_grained_lineage.append(
                         FineGrainedLineage(
                             upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
@@ -161,7 +171,19 @@ def _extend_lineage(self, connector: Connector, datajob: DataJob) -> None:
         datajob.inlets.extend(input_dataset_urn_list)
         datajob.outlets.extend(output_dataset_urn_list)
         datajob.fine_grained_lineages.extend(fine_grained_lineage)
-        return None
+
+        return dict(
+            **{
+                f"source.{k}": str(v)
+                for k, v in source_details.dict().items()
+                if v is not None
+            },
+            **{
+                f"destination.{k}": str(v)
+                for k, v in destination_details.dict().items()
+                if v is not None
+            },
+        )
 
     def _generate_dataflow_from_connector(self, connector: Connector) -> DataFlow:
         return DataFlow(
@@ -187,23 +209,23 @@ def _generate_datajob_from_connector(self, connector: Connector) -> DataJob:
             owners={owner_email} if owner_email else set(),
         )
 
-        job_property_bag: Dict[str, str] = {}
-        allowed_connection_keys = [
-            Constant.PAUSED,
-            Constant.SYNC_FREQUENCY,
-            Constant.DESTINATION_ID,
-        ]
-        for key in allowed_connection_keys:
-            if hasattr(connector, key) and getattr(connector, key) is not None:
-                job_property_bag[key] = repr(getattr(connector, key))
-        datajob.properties = job_property_bag
-
         # Map connector source and destination table with dataset entity
         # Also extend the fine grained lineage of column if include_column_lineage is True
-        self._extend_lineage(connector=connector, datajob=datajob)
-
+        lineage_properties = self._extend_lineage(connector=connector, datajob=datajob)
         # TODO: Add fine grained lineages of dataset after FineGrainedLineageDownstreamType.DATASET enabled
 
+        connector_properties: Dict[str, str] = {
+            "connector_id": connector.connector_id,
+            "connector_type": connector.connector_type,
+            "paused": str(connector.paused),
+            "sync_frequency": str(connector.sync_frequency),
+            "destination_id": connector.destination_id,
+        }
+        datajob.properties = {
+            **connector_properties,
+            **lineage_properties,
+        }
+
         return datajob
 
     def _generate_dpi_from_job(self, job: Job, datajob: DataJob) -> DataProcessInstance:
@@ -257,16 +279,14 @@ def _get_connector_workunits(
         for mcp in datajob.generate_mcp(materialize_iolets=False):
             yield mcp.as_workunit()
 
-        # Materialize the upstream referenced datasets.
-        # We assume that the downstreams are materialized by other ingestion sources.
-        for iolet in datajob.inlets:
-            # We don't want these to be tracked by stateful ingestion.
-            yield MetadataChangeProposalWrapper(
-                entityUrn=str(iolet),
-                aspect=StatusClass(removed=False),
-            ).as_workunit(is_primary_source=False)
-
         # Map Fivetran's job/sync history entity with Datahub's data process entity
+        if len(connector.jobs) >= MAX_JOBS_PER_CONNECTOR:
+            self.report.warning(
+                title="Not all sync history was captured",
+                message=f"The connector had more than {MAX_JOBS_PER_CONNECTOR} sync runs in the past {self.config.history_sync_lookback_period} days. "
+                f"Only the most recent {MAX_JOBS_PER_CONNECTOR} syncs were ingested.",
+                context=f"{connector.connector_name} (connector_id: {connector.connector_id})",
+            )
         for job in connector.jobs:
             dpi = self._generate_dpi_from_job(job, datajob)
             yield from self._get_dpi_workunits(job, dpi)
@@ -279,7 +299,9 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source:
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
             *super().get_workunit_processors(),
-            self.stale_entity_removal_handler.workunit_processor,
+            StaleEntityRemovalHandler.create(
+                self, self.config, self.ctx
+            ).workunit_processor,
         ]
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
index 31c16139066e4..529002270cdd9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
@@ -1,6 +1,7 @@
 import functools
 import json
 import logging
+from collections import defaultdict
 from typing import Any, Dict, List, Optional, Tuple
 
 import sqlglot
@@ -83,59 +84,59 @@ def _query(self, query: str) -> List[Dict]:
             query = sqlglot.parse_one(query, dialect="snowflake").sql(
                 dialect=self.fivetran_log_config.destination_platform, pretty=True
             )
-        logger.debug(f"Query : {query}")
+        logger.info(f"Executing query: {query}")
         resp = self.engine.execute(query)
         return [row for row in resp]
 
-    def _get_column_lineage_metadata(self) -> Dict[str, List]:
+    def _get_column_lineage_metadata(
+        self, connector_ids: List[str]
+    ) -> Dict[Tuple[str, str], List]:
         """
-        Return's dict of column lineage metadata with key as '<SOURCE_TABLE_ID>-<DESTINATION_TABLE_ID>'
+        Returns dict of column lineage metadata with key as (<SOURCE_TABLE_ID>, <DESTINATION_TABLE_ID>)
         """
-        all_column_lineage: Dict[str, List] = {}
+        all_column_lineage = defaultdict(list)
         column_lineage_result = self._query(
-            self.fivetran_log_query.get_column_lineage_query()
+            self.fivetran_log_query.get_column_lineage_query(
+                connector_ids=connector_ids
+            )
         )
         for column_lineage in column_lineage_result:
-            key = f"{column_lineage[Constant.SOURCE_TABLE_ID]}-{column_lineage[Constant.DESTINATION_TABLE_ID]}"
-            if key not in all_column_lineage:
-                all_column_lineage[key] = [column_lineage]
-            else:
-                all_column_lineage[key].append(column_lineage)
-        return all_column_lineage
+            key = (
+                column_lineage[Constant.SOURCE_TABLE_ID],
+                column_lineage[Constant.DESTINATION_TABLE_ID],
+            )
+            all_column_lineage[key].append(column_lineage)
+        return dict(all_column_lineage)
 
-    def _get_connectors_table_lineage_metadata(self) -> Dict[str, List]:
+    def _get_table_lineage_metadata(self, connector_ids: List[str]) -> Dict[str, List]:
         """
-        Return's dict of table lineage metadata with key as 'CONNECTOR_ID'
+        Returns dict of table lineage metadata with key as 'CONNECTOR_ID'
         """
-        connectors_table_lineage_metadata: Dict[str, List] = {}
+        connectors_table_lineage_metadata = defaultdict(list)
         table_lineage_result = self._query(
-            self.fivetran_log_query.get_table_lineage_query()
+            self.fivetran_log_query.get_table_lineage_query(connector_ids=connector_ids)
         )
         for table_lineage in table_lineage_result:
-            if (
+            connectors_table_lineage_metadata[
                 table_lineage[Constant.CONNECTOR_ID]
-                not in connectors_table_lineage_metadata
-            ):
-                connectors_table_lineage_metadata[
-                    table_lineage[Constant.CONNECTOR_ID]
-                ] = [table_lineage]
-            else:
-                connectors_table_lineage_metadata[
-                    table_lineage[Constant.CONNECTOR_ID]
-                ].append(table_lineage)
-        return connectors_table_lineage_metadata
+            ].append(table_lineage)
+        return dict(connectors_table_lineage_metadata)
 
-    def _get_table_lineage(
+    def _extract_connector_lineage(
         self,
-        column_lineage_metadata: Dict[str, List],
         table_lineage_result: Optional[List],
+        column_lineage_metadata: Dict[Tuple[str, str], List],
     ) -> List[TableLineage]:
         table_lineage_list: List[TableLineage] = []
         if table_lineage_result is None:
             return table_lineage_list
         for table_lineage in table_lineage_result:
+            # Join the column lineage into the table lineage.
             column_lineage_result = column_lineage_metadata.get(
-                f"{table_lineage[Constant.SOURCE_TABLE_ID]}-{table_lineage[Constant.DESTINATION_TABLE_ID]}"
+                (
+                    table_lineage[Constant.SOURCE_TABLE_ID],
+                    table_lineage[Constant.DESTINATION_TABLE_ID],
+                )
             )
             column_lineage_list: List[ColumnLineage] = []
             if column_lineage_result:
@@ -148,6 +149,7 @@ def _get_table_lineage(
                     )
                     for column_lineage in column_lineage_result
                 ]
+
             table_lineage_list.append(
                 TableLineage(
                     source_table=f"{table_lineage[Constant.SOURCE_SCHEMA_NAME]}.{table_lineage[Constant.SOURCE_TABLE_NAME]}",
@@ -158,34 +160,27 @@ def _get_table_lineage(
 
         return table_lineage_list
 
-    def _get_all_connector_sync_logs(self, syncs_interval: int) -> Dict[str, Dict]:
-        sync_logs = {}
-        for row in self._query(
-            self.fivetran_log_query.get_sync_logs_query().format(
-                db_clause=self.fivetran_log_query.db_clause,
-                syncs_interval=syncs_interval,
-            )
-        ):
-            if row[Constant.CONNECTOR_ID] not in sync_logs:
-                sync_logs[row[Constant.CONNECTOR_ID]] = {
-                    row[Constant.SYNC_ID]: {
-                        row["message_event"]: (
-                            row[Constant.TIME_STAMP].timestamp(),
-                            row[Constant.MESSAGE_DATA],
-                        )
-                    }
-                }
-            elif row[Constant.SYNC_ID] not in sync_logs[row[Constant.CONNECTOR_ID]]:
-                sync_logs[row[Constant.CONNECTOR_ID]][row[Constant.SYNC_ID]] = {
-                    row["message_event"]: (
-                        row[Constant.TIME_STAMP].timestamp(),
-                        row[Constant.MESSAGE_DATA],
-                    )
-                }
-            else:
-                sync_logs[row[Constant.CONNECTOR_ID]][row[Constant.SYNC_ID]][
-                    row["message_event"]
-                ] = (row[Constant.TIME_STAMP].timestamp(), row[Constant.MESSAGE_DATA])
+    def _get_all_connector_sync_logs(
+        self, syncs_interval: int, connector_ids: List[str]
+    ) -> Dict[str, Dict[str, Dict[str, Tuple[float, Optional[str]]]]]:
+        sync_logs: Dict[str, Dict[str, Dict[str, Tuple[float, Optional[str]]]]] = {}
+
+        query = self.fivetran_log_query.get_sync_logs_query(
+            syncs_interval=syncs_interval,
+            connector_ids=connector_ids,
+        )
+
+        for row in self._query(query):
+            connector_id = row[Constant.CONNECTOR_ID]
+            sync_id = row[Constant.SYNC_ID]
+
+            if connector_id not in sync_logs:
+                sync_logs[connector_id] = {}
+
+            sync_logs[connector_id][sync_id] = {
+                "sync_start": (row["start_time"].timestamp(), None),
+                "sync_end": (row["end_time"].timestamp(), row["end_message_data"]),
+            }
 
         return sync_logs
 
@@ -232,19 +227,23 @@ def get_user_email(self, user_id: str) -> Optional[str]:
             return None
         return self._get_users().get(user_id)
 
-    def _fill_connectors_table_lineage(self, connectors: List[Connector]) -> None:
-        table_lineage_metadata = self._get_connectors_table_lineage_metadata()
-        column_lineage_metadata = self._get_column_lineage_metadata()
+    def _fill_connectors_lineage(self, connectors: List[Connector]) -> None:
+        connector_ids = [connector.connector_id for connector in connectors]
+        table_lineage_metadata = self._get_table_lineage_metadata(connector_ids)
+        column_lineage_metadata = self._get_column_lineage_metadata(connector_ids)
         for connector in connectors:
-            connector.table_lineage = self._get_table_lineage(
-                column_lineage_metadata=column_lineage_metadata,
+            connector.lineage = self._extract_connector_lineage(
                 table_lineage_result=table_lineage_metadata.get(connector.connector_id),
+                column_lineage_metadata=column_lineage_metadata,
             )
 
     def _fill_connectors_jobs(
         self, connectors: List[Connector], syncs_interval: int
     ) -> None:
-        sync_logs = self._get_all_connector_sync_logs(syncs_interval)
+        connector_ids = [connector.connector_id for connector in connectors]
+        sync_logs = self._get_all_connector_sync_logs(
+            syncs_interval, connector_ids=connector_ids
+        )
         for connector in connectors:
             connector.jobs = self._get_jobs_list(sync_logs.get(connector.connector_id))
 
@@ -257,29 +256,48 @@ def get_allowed_connectors_list(
     ) -> List[Connector]:
         connectors: List[Connector] = []
         with report.metadata_extraction_perf.connectors_metadata_extraction_sec:
+            logger.info("Fetching connector list")
             connector_list = self._query(self.fivetran_log_query.get_connectors_query())
             for connector in connector_list:
-                if not connector_patterns.allowed(connector[Constant.CONNECTOR_NAME]):
-                    report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME])
+                connector_id = connector[Constant.CONNECTOR_ID]
+                connector_name = connector[Constant.CONNECTOR_NAME]
+                if not connector_patterns.allowed(connector_name):
+                    report.report_connectors_dropped(
+                        f"{connector_name} (connector_id: {connector_id}, dropped due to filter pattern)"
+                    )
                     continue
-                if not destination_patterns.allowed(connector[Constant.DESTINATION_ID]):
-                    report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME])
+                if not destination_patterns.allowed(
+                    destination_id := connector[Constant.DESTINATION_ID]
+                ):
+                    report.report_connectors_dropped(
+                        f"{connector_name} (connector_id: {connector_id}, destination_id: {destination_id})"
+                    )
                     continue
                 connectors.append(
                     Connector(
-                        connector_id=connector[Constant.CONNECTOR_ID],
-                        connector_name=connector[Constant.CONNECTOR_NAME],
+                        connector_id=connector_id,
+                        connector_name=connector_name,
                         connector_type=connector[Constant.CONNECTOR_TYPE_ID],
                         paused=connector[Constant.PAUSED],
                         sync_frequency=connector[Constant.SYNC_FREQUENCY],
-                        destination_id=connector[Constant.DESTINATION_ID],
+                        destination_id=destination_id,
                         user_id=connector[Constant.CONNECTING_USER_ID],
-                        table_lineage=[],
-                        jobs=[],
+                        lineage=[],  # filled later
+                        jobs=[],  # filled later
                     )
                 )
+
+        if not connectors:
+            # Some of our queries don't work well when there's no connectors, since
+            # we push down connector id filters.
+            logger.info("No allowed connectors found")
+            return []
+        logger.info(f"Found {len(connectors)} allowed connectors")
+
         with report.metadata_extraction_perf.connectors_lineage_extraction_sec:
-            self._fill_connectors_table_lineage(connectors)
+            logger.info("Fetching connector lineage")
+            self._fill_connectors_lineage(connectors)
         with report.metadata_extraction_perf.connectors_jobs_extraction_sec:
+            logger.info("Fetching connector job run history")
             self._fill_connectors_jobs(connectors, syncs_interval)
         return connectors
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py
index d965f53ff554b..65378928b244d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py
@@ -1,3 +1,11 @@
+from typing import List
+
+# Safeguards to prevent fetching massive amounts of data.
+MAX_TABLE_LINEAGE_PER_CONNECTOR = 120
+MAX_COLUMN_LINEAGE_PER_CONNECTOR = 1000
+MAX_JOBS_PER_CONNECTOR = 500
+
+
 class FivetranLogQuery:
     # Note: All queries are written in Snowflake SQL.
     # They will be transpiled to the target database's SQL dialect at runtime.
@@ -24,51 +32,117 @@ def get_connectors_query(self) -> str:
   destination_id
 FROM {self.db_clause}connector
 WHERE
-  _fivetran_deleted = FALSE\
+  _fivetran_deleted = FALSE
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY _fivetran_synced DESC) = 1
 """
 
     def get_users_query(self) -> str:
-        return f"""
-        SELECT id as user_id,
-        given_name,
-        family_name,
-        email
-        FROM {self.db_clause}user"""
+        return f"""\
+SELECT id as user_id,
+given_name,
+family_name,
+email
+FROM {self.db_clause}user
+"""
 
-    def get_sync_logs_query(self) -> str:
-        return """
-        SELECT connector_id,
+    def get_sync_logs_query(
+        self,
+        syncs_interval: int,
+        connector_ids: List[str],
+    ) -> str:
+        # Format connector_ids as a comma-separated string of quoted IDs
+        formatted_connector_ids = ", ".join(f"'{id}'" for id in connector_ids)
+
+        return f"""\
+WITH ranked_syncs AS (
+    SELECT
+        connector_id,
         sync_id,
-        message_event,
-        message_data,
-        time_stamp
-        FROM {db_clause}log
-        WHERE message_event in ('sync_start', 'sync_end')
-        and time_stamp > CURRENT_TIMESTAMP - INTERVAL '{syncs_interval} days'"""
+        MAX(CASE WHEN message_event = 'sync_start' THEN time_stamp END) as start_time,
+        MAX(CASE WHEN message_event = 'sync_end' THEN time_stamp END) as end_time,
+        MAX(CASE WHEN message_event = 'sync_end' THEN message_data END) as end_message_data,
+        ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY MAX(time_stamp) DESC) as rn
+    FROM {self.db_clause}log
+    WHERE message_event in ('sync_start', 'sync_end')
+    AND time_stamp > CURRENT_TIMESTAMP - INTERVAL '{syncs_interval} days'
+    AND connector_id IN ({formatted_connector_ids})
+    GROUP BY connector_id, sync_id
+)
+SELECT
+    connector_id,
+    sync_id,
+    start_time,
+    end_time,
+    end_message_data
+FROM ranked_syncs
+WHERE rn <= {MAX_JOBS_PER_CONNECTOR}
+    AND start_time IS NOT NULL
+    AND end_time IS NOT NULL
+ORDER BY connector_id, end_time DESC
+"""
 
-    def get_table_lineage_query(self) -> str:
-        return f"""
-        SELECT stm.connector_id as connector_id,
+    def get_table_lineage_query(self, connector_ids: List[str]) -> str:
+        # Format connector_ids as a comma-separated string of quoted IDs
+        formatted_connector_ids = ", ".join(f"'{id}'" for id in connector_ids)
+
+        return f"""\
+SELECT
+    *
+FROM (
+    SELECT
+        stm.connector_id as connector_id,
         stm.id as source_table_id,
         stm.name as source_table_name,
         ssm.name as source_schema_name,
         dtm.id as destination_table_id,
         dtm.name as destination_table_name,
-        dsm.name as destination_schema_name
-        FROM {self.db_clause}table_lineage as tl
-        JOIN {self.db_clause}source_table_metadata as stm on tl.source_table_id = stm.id
-        JOIN {self.db_clause}destination_table_metadata as dtm on tl.destination_table_id = dtm.id
-        JOIN {self.db_clause}source_schema_metadata as ssm on stm.schema_id = ssm.id
-        JOIN {self.db_clause}destination_schema_metadata as dsm on dtm.schema_id = dsm.id"""
+        dsm.name as destination_schema_name,
+        tl.created_at as created_at,
+        ROW_NUMBER() OVER (PARTITION BY stm.connector_id, stm.id, dtm.id ORDER BY tl.created_at DESC) as table_combo_rn
+    FROM {self.db_clause}table_lineage as tl
+    JOIN {self.db_clause}source_table_metadata as stm on tl.source_table_id = stm.id
+    JOIN {self.db_clause}destination_table_metadata as dtm on tl.destination_table_id = dtm.id
+    JOIN {self.db_clause}source_schema_metadata as ssm on stm.schema_id = ssm.id
+    JOIN {self.db_clause}destination_schema_metadata as dsm on dtm.schema_id = dsm.id
+    WHERE stm.connector_id IN ({formatted_connector_ids})
+)
+-- Ensure that we only get back one entry per source and destination pair.
+WHERE table_combo_rn = 1
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY created_at DESC) <= {MAX_TABLE_LINEAGE_PER_CONNECTOR}
+ORDER BY connector_id, created_at DESC
+"""
 
-    def get_column_lineage_query(self) -> str:
-        return f"""
-        SELECT scm.table_id as source_table_id,
+    def get_column_lineage_query(self, connector_ids: List[str]) -> str:
+        # Format connector_ids as a comma-separated string of quoted IDs
+        formatted_connector_ids = ", ".join(f"'{id}'" for id in connector_ids)
+
+        return f"""\
+SELECT
+    source_table_id,
+    destination_table_id,
+    source_column_name,
+    destination_column_name
+FROM (
+    SELECT
+        stm.connector_id as connector_id,
+        scm.table_id as source_table_id,
         dcm.table_id as destination_table_id,
         scm.name as source_column_name,
-        dcm.name as destination_column_name
-        FROM {self.db_clause}column_lineage as cl
-        JOIN {self.db_clause}source_column_metadata as scm
-        on cl.source_column_id = scm.id
-        JOIN {self.db_clause}destination_column_metadata as dcm
-        on cl.destination_column_id = dcm.id"""
+        dcm.name as destination_column_name,
+        cl.created_at as created_at,
+        ROW_NUMBER() OVER (PARTITION BY stm.connector_id, cl.source_column_id, cl.destination_column_id ORDER BY cl.created_at DESC) as column_combo_rn
+    FROM {self.db_clause}column_lineage as cl
+    JOIN {self.db_clause}source_column_metadata as scm
+      ON cl.source_column_id = scm.id
+    JOIN {self.db_clause}destination_column_metadata as dcm
+      ON cl.destination_column_id = dcm.id
+    -- Only joining source_table_metadata to get the connector_id.
+    JOIN {self.db_clause}source_table_metadata as stm
+      ON scm.table_id = stm.id
+    WHERE stm.connector_id IN ({formatted_connector_ids})
+)
+-- Ensure that we only get back one entry per (connector, source column, destination column) pair.
+WHERE column_combo_rn = 1
+QUALIFY ROW_NUMBER() OVER (PARTITION BY connector_id ORDER BY created_at DESC) <= {MAX_COLUMN_LINEAGE_PER_CONNECTOR}
+ORDER BY connector_id, created_at DESC
+"""
diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py
index 1897f3f288ec0..4eecbb4d9d717 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py
@@ -24,6 +24,11 @@
     DataProcessCleanupConfig,
     DataProcessCleanupReport,
 )
+from datahub.ingestion.source.gc.execution_request_cleanup import (
+    DatahubExecutionRequestCleanup,
+    DatahubExecutionRequestCleanupConfig,
+    DatahubExecutionRequestCleanupReport,
+)
 from datahub.ingestion.source.gc.soft_deleted_entity_cleanup import (
     SoftDeletedEntitiesCleanup,
     SoftDeletedEntitiesCleanupConfig,
@@ -60,19 +65,28 @@ class DataHubGcSourceConfig(ConfigModel):
         description="Sleep between truncation monitoring.",
     )
 
-    dataprocess_cleanup: Optional[DataProcessCleanupConfig] = Field(
-        default=None,
+    dataprocess_cleanup: DataProcessCleanupConfig = Field(
+        default_factory=DataProcessCleanupConfig,
         description="Configuration for data process cleanup",
     )
 
-    soft_deleted_entities_cleanup: Optional[SoftDeletedEntitiesCleanupConfig] = Field(
-        default=None,
+    soft_deleted_entities_cleanup: SoftDeletedEntitiesCleanupConfig = Field(
+        default_factory=SoftDeletedEntitiesCleanupConfig,
         description="Configuration for soft deleted entities cleanup",
     )
 
+    execution_request_cleanup: DatahubExecutionRequestCleanupConfig = Field(
+        default_factory=DatahubExecutionRequestCleanupConfig,
+        description="Configuration for execution request cleanup",
+    )
+
 
 @dataclass
-class DataHubGcSourceReport(DataProcessCleanupReport, SoftDeletedEntitiesReport):
+class DataHubGcSourceReport(
+    DataProcessCleanupReport,
+    SoftDeletedEntitiesReport,
+    DatahubExecutionRequestCleanupReport,
+):
     expired_tokens_revoked: int = 0
 
 
@@ -94,21 +108,22 @@ def __init__(self, ctx: PipelineContext, config: DataHubGcSourceConfig):
         self.ctx = ctx
         self.config = config
         self.report = DataHubGcSourceReport()
+        self.report.event_not_produced_warn = False
         self.graph = ctx.require_graph("The DataHubGc source")
-        self.dataprocess_cleanup: Optional[DataProcessCleanup] = None
-        self.soft_deleted_entities_cleanup: Optional[SoftDeletedEntitiesCleanup] = None
-
-        if self.config.dataprocess_cleanup:
-            self.dataprocess_cleanup = DataProcessCleanup(
-                ctx, self.config.dataprocess_cleanup, self.report, self.config.dry_run
-            )
-        if self.config.soft_deleted_entities_cleanup:
-            self.soft_deleted_entities_cleanup = SoftDeletedEntitiesCleanup(
-                ctx,
-                self.config.soft_deleted_entities_cleanup,
-                self.report,
-                self.config.dry_run,
-            )
+        self.dataprocess_cleanup = DataProcessCleanup(
+            ctx, self.config.dataprocess_cleanup, self.report, self.config.dry_run
+        )
+        self.soft_deleted_entities_cleanup = SoftDeletedEntitiesCleanup(
+            ctx,
+            self.config.soft_deleted_entities_cleanup,
+            self.report,
+            self.config.dry_run,
+        )
+        self.execution_request_cleanup = DatahubExecutionRequestCleanup(
+            config=self.config.execution_request_cleanup,
+            graph=self.graph,
+            report=self.report,
+        )
 
     @classmethod
     def create(cls, config_dict, ctx):
@@ -123,13 +138,32 @@ def get_workunits_internal(
         self,
     ) -> Iterable[MetadataWorkUnit]:
         if self.config.cleanup_expired_tokens:
-            self.revoke_expired_tokens()
+            try:
+                self.revoke_expired_tokens()
+            except Exception as e:
+                self.report.failure("While trying to cleanup expired token ", exc=e)
         if self.config.truncate_indices:
-            self.truncate_indices()
-        if self.dataprocess_cleanup:
-            yield from self.dataprocess_cleanup.get_workunits_internal()
-        if self.soft_deleted_entities_cleanup:
-            self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities()
+            try:
+                self.truncate_indices()
+            except Exception as e:
+                self.report.failure("While trying to truncate indices ", exc=e)
+        if self.config.soft_deleted_entities_cleanup.enabled:
+            try:
+                self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities()
+            except Exception as e:
+                self.report.failure(
+                    "While trying to cleanup soft deleted entities ", exc=e
+                )
+        if self.config.execution_request_cleanup.enabled:
+            try:
+                self.execution_request_cleanup.run()
+            except Exception as e:
+                self.report.failure("While trying to cleanup execution request ", exc=e)
+        if self.config.dataprocess_cleanup.enabled:
+            try:
+                yield from self.dataprocess_cleanup.get_workunits_internal()
+            except Exception as e:
+                self.report.failure("While trying to cleanup data process ", exc=e)
         yield from []
 
     def truncate_indices(self) -> None:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py
index 80f7b7a9f4480..6d16aaab2d798 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py
@@ -98,6 +98,9 @@
 
 
 class DataProcessCleanupConfig(ConfigModel):
+    enabled: bool = Field(
+        default=True, description="Whether to do data process cleanup."
+    )
     retention_days: Optional[int] = Field(
         10,
         description="Number of days to retain metadata in DataHub",
@@ -114,11 +117,11 @@ class DataProcessCleanupConfig(ConfigModel):
     )
 
     delete_empty_data_jobs: bool = Field(
-        True, description="Wether to delete Data Jobs without runs"
+        False, description="Whether to delete Data Jobs without runs"
     )
 
     delete_empty_data_flows: bool = Field(
-        True, description="Wether to delete Data Flows without runs"
+        False, description="Whether to delete Data Flows without runs"
     )
 
     hard_delete_entities: bool = Field(
@@ -128,7 +131,7 @@ class DataProcessCleanupConfig(ConfigModel):
 
     batch_size: int = Field(
         500,
-        description="The number of entities to get in a batch from GraphQL",
+        description="The number of entities to get in a batch from API",
     )
 
     max_workers: int = Field(
@@ -173,9 +176,9 @@ class DataProcessCleanup:
     """
     This source is a maintenance source which cleans up old/unused aspects.
 
-    Currently it only supports:.
+    Currently it only supports:
         - DataFlow
-        -DataJob
+        - DataJob
         - DataProcessInstance
 
     """
@@ -207,23 +210,34 @@ def fetch_dpis(self, job_urn: str, batch_size: int) -> List[dict]:
         assert self.ctx.graph
         dpis = []
         start = 0
+        # This graphql endpoint doesn't support scrolling and therefore after 10k DPIs it causes performance issues on ES
+        # Therefore, we are limiting the max DPIs to 9000
+        max_item = 9000
         while True:
-            job_query_result = self.ctx.graph.execute_graphql(
-                DATA_PROCESS_INSTANCES_QUERY,
-                {"dataJobUrn": job_urn, "start": start, "count": batch_size},
-            )
-            job_data = job_query_result.get("dataJob")
-            if not job_data:
-                raise ValueError(f"Error getting job {job_urn}")
-
-            runs_data = job_data.get("runs")
-            if not runs_data:
-                raise ValueError(f"Error getting runs for {job_urn}")
-
-            runs = runs_data.get("runs")
-            dpis.extend(runs)
-            start += batch_size
-            if len(runs) < batch_size:
+            try:
+                job_query_result = self.ctx.graph.execute_graphql(
+                    DATA_PROCESS_INSTANCES_QUERY,
+                    {"dataJobUrn": job_urn, "start": start, "count": batch_size},
+                )
+                job_data = job_query_result.get("dataJob")
+                if not job_data:
+                    logger.error(f"Error getting job {job_urn}")
+                    break
+
+                runs_data = job_data.get("runs")
+                if not runs_data:
+                    logger.error(f"Error getting runs for {job_urn}")
+                    break
+
+                runs = runs_data.get("runs")
+                dpis.extend(runs)
+                start += batch_size
+                if len(runs) < batch_size or start >= max_item:
+                    break
+            except Exception as e:
+                self.report.failure(
+                    f"Exception while fetching DPIs for job {job_urn}:", exc=e
+                )
                 break
         return dpis
 
@@ -243,9 +257,14 @@ def keep_last_n_dpi(
                 futures[future] = dpi
 
             for future in as_completed(futures):
-                deleted_count_last_n += 1
-                futures[future]["deleted"] = True
-
+                try:
+                    future.result()
+                    deleted_count_last_n += 1
+                    futures[future]["deleted"] = True
+                except Exception as e:
+                    self.report.report_failure(
+                        f"Exception while deleting DPI: {e}", exc=e
+                    )
             if deleted_count_last_n % self.config.batch_size == 0:
                 logger.info(f"Deleted {deleted_count_last_n} DPIs from {job.urn}")
                 if self.config.delay:
@@ -267,7 +286,7 @@ def delete_entity(self, urn: str, type: str) -> None:
 
         if self.dry_run:
             logger.info(
-                f"Dry run is on otherwise it would have deleted {urn} with hard deletion is{self.config.hard_delete_entities}"
+                f"Dry run is on otherwise it would have deleted {urn} with hard deletion is {self.config.hard_delete_entities}"
             )
             return
 
@@ -277,7 +296,12 @@ def delete_dpi_from_datajobs(self, job: DataJobEntity) -> None:
         assert self.ctx.graph
 
         dpis = self.fetch_dpis(job.urn, self.config.batch_size)
-        dpis.sort(key=lambda x: x["created"]["time"], reverse=True)
+        dpis.sort(
+            key=lambda x: x["created"]["time"]
+            if x.get("created") and x["created"].get("time")
+            else 0,
+            reverse=True,
+        )
 
         with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor:
             if self.config.keep_last_n:
@@ -309,15 +333,23 @@ def remove_old_dpis(
             if dpi.get("deleted"):
                 continue
 
-            if dpi["created"]["time"] < retention_time * 1000:
+            if (
+                not dpi.get("created")
+                or not dpi["created"].get("time")
+                or dpi["created"]["time"] < retention_time * 1000
+            ):
                 future = executor.submit(
                     self.delete_entity, dpi["urn"], "dataprocessInstance"
                 )
                 futures[future] = dpi
 
         for future in as_completed(futures):
-            deleted_count_retention += 1
-            futures[future]["deleted"] = True
+            try:
+                future.result()
+                deleted_count_retention += 1
+                futures[future]["deleted"] = True
+            except Exception as e:
+                self.report.report_failure(f"Exception while deleting DPI: {e}", exc=e)
 
             if deleted_count_retention % self.config.batch_size == 0:
                 logger.info(
@@ -328,9 +360,12 @@ def remove_old_dpis(
                     logger.info(f"Sleeping for {self.config.delay} seconds")
                     time.sleep(self.config.delay)
 
-        logger.info(
-            f"Deleted {deleted_count_retention} DPIs from {job.urn} due to retention"
-        )
+        if deleted_count_retention > 0:
+            logger.info(
+                f"Deleted {deleted_count_retention} DPIs from {job.urn} due to retention"
+            )
+        else:
+            logger.debug(f"No DPIs to delete from {job.urn} due to retention")
 
     def get_data_flows(self) -> Iterable[DataFlowEntity]:
         assert self.ctx.graph
@@ -339,17 +374,26 @@ def get_data_flows(self) -> Iterable[DataFlowEntity]:
         previous_scroll_id: Optional[str] = None
 
         while True:
-            result = self.ctx.graph.execute_graphql(
-                DATAFLOW_QUERY,
-                {
-                    "query": "*",
-                    "scrollId": scroll_id if scroll_id else None,
-                    "batchSize": self.config.batch_size,
-                },
-            )
+            result = None
+            try:
+                result = self.ctx.graph.execute_graphql(
+                    DATAFLOW_QUERY,
+                    {
+                        "query": "*",
+                        "scrollId": scroll_id if scroll_id else None,
+                        "batchSize": self.config.batch_size,
+                    },
+                )
+            except Exception as e:
+                self.report.failure(
+                    f"While trying to get dataflows with {scroll_id}", exc=e
+                )
+                break
+
             scrollAcrossEntities = result.get("scrollAcrossEntities")
             if not scrollAcrossEntities:
                 raise ValueError("Missing scrollAcrossEntities in response")
+            logger.info(f"Got {scrollAcrossEntities.get('count')} DataFlow entities")
 
             scroll_id = scrollAcrossEntities.get("nextScrollId")
             for flow in scrollAcrossEntities.get("searchResults"):
@@ -366,6 +410,8 @@ def get_data_flows(self) -> Iterable[DataFlowEntity]:
             previous_scroll_id = scroll_id
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        if not self.config.enabled:
+            return []
         assert self.ctx.graph
 
         dataFlows: Dict[str, DataFlowEntity] = {}
@@ -373,17 +419,26 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             dataFlows[flow.urn] = flow
 
         scroll_id: Optional[str] = None
+        previous_scroll_id: Optional[str] = None
+
         dataJobs: Dict[str, List[DataJobEntity]] = defaultdict(list)
         deleted_jobs: int = 0
+
         while True:
-            result = self.ctx.graph.execute_graphql(
-                DATAJOB_QUERY,
-                {
-                    "query": "*",
-                    "scrollId": scroll_id if scroll_id else None,
-                    "batchSize": self.config.batch_size,
-                },
-            )
+            try:
+                result = self.ctx.graph.execute_graphql(
+                    DATAJOB_QUERY,
+                    {
+                        "query": "*",
+                        "scrollId": scroll_id if scroll_id else None,
+                        "batchSize": self.config.batch_size,
+                    },
+                )
+            except Exception as e:
+                self.report.failure(
+                    f"While trying to get data jobs with {scroll_id}", exc=e
+                )
+                break
             scrollAcrossEntities = result.get("scrollAcrossEntities")
             if not scrollAcrossEntities:
                 raise ValueError("Missing scrollAcrossEntities in response")
@@ -401,7 +456,12 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     total_runs=job.get("entity").get("runs").get("total"),
                 )
                 if datajob_entity.total_runs > 0:
-                    self.delete_dpi_from_datajobs(datajob_entity)
+                    try:
+                        self.delete_dpi_from_datajobs(datajob_entity)
+                    except Exception as e:
+                        self.report.failure(
+                            f"While trying to delete {datajob_entity} ", exc=e
+                        )
                 if (
                     datajob_entity.total_runs == 0
                     and self.config.delete_empty_data_jobs
@@ -416,9 +476,11 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 else:
                     dataJobs[datajob_entity.flow_urn].append(datajob_entity)
 
-            if not scroll_id:
+            if not scroll_id or previous_scroll_id == scroll_id:
                 break
 
+            previous_scroll_id = scroll_id
+
         logger.info(f"Deleted {deleted_jobs} DataJobs")
         # Delete empty dataflows if needed
         if self.config.delete_empty_data_flows:
@@ -433,4 +495,5 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     if deleted_jobs % self.config.batch_size == 0:
                         logger.info(f"Deleted {deleted_data_flows} DataFlows")
             logger.info(f"Deleted {deleted_data_flows} DataFlows")
+
         return []
diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py
new file mode 100644
index 0000000000000..3baf858e44cdc
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py
@@ -0,0 +1,239 @@
+import logging
+import time
+from typing import Any, Dict, Iterator, Optional
+
+from pydantic import BaseModel, Field
+
+from datahub.configuration.common import ConfigModel
+from datahub.ingestion.api.source import SourceReport
+from datahub.ingestion.graph.client import DataHubGraph
+
+logger = logging.getLogger(__name__)
+
+DATAHUB_EXECUTION_REQUEST_ENTITY_NAME = "dataHubExecutionRequest"
+DATAHUB_EXECUTION_REQUEST_KEY_ASPECT_NAME = "dataHubExecutionRequestKey"
+DATAHUB_EXECUTION_REQUEST_INPUT_ASPECT_NAME = "dataHubExecutionRequestInput"
+DATAHUB_EXECUTION_REQUEST_RESULT_ASPECT_NAME = "dataHubExecutionRequestResult"
+
+
+class DatahubExecutionRequestCleanupConfig(ConfigModel):
+    keep_history_min_count: int = Field(
+        10,
+        description="Minimum number of execution requests to keep, per ingestion source",
+    )
+
+    keep_history_max_count: int = Field(
+        1000,
+        description="Maximum number of execution requests to keep, per ingestion source",
+    )
+
+    keep_history_max_days: int = Field(
+        30,
+        description="Maximum number of days to keep execution requests for, per ingestion source",
+    )
+
+    batch_read_size: int = Field(
+        100,
+        description="Number of records per read operation",
+    )
+
+    enabled: bool = Field(
+        True,
+        description="Global switch for this cleanup task",
+    )
+
+    def keep_history_max_milliseconds(self):
+        return self.keep_history_max_days * 24 * 3600 * 1000
+
+
+class DatahubExecutionRequestCleanupReport(SourceReport):
+    execution_request_cleanup_records_read: int = 0
+    execution_request_cleanup_records_preserved: int = 0
+    execution_request_cleanup_records_deleted: int = 0
+    execution_request_cleanup_read_errors: int = 0
+    execution_request_cleanup_delete_errors: int = 0
+
+
+class CleanupRecord(BaseModel):
+    urn: str
+    request_id: str
+    status: str
+    ingestion_source: str
+    requested_at: int
+
+
+class DatahubExecutionRequestCleanup:
+    def __init__(
+        self,
+        graph: DataHubGraph,
+        report: DatahubExecutionRequestCleanupReport,
+        config: Optional[DatahubExecutionRequestCleanupConfig] = None,
+    ) -> None:
+        self.graph = graph
+        self.report = report
+        self.instance_id = int(time.time())
+
+        if config is not None:
+            self.config = config
+        else:
+            self.config = DatahubExecutionRequestCleanupConfig()
+
+    def _to_cleanup_record(self, entry: Dict) -> CleanupRecord:
+        input_aspect = (
+            entry.get("aspects", {})
+            .get(DATAHUB_EXECUTION_REQUEST_INPUT_ASPECT_NAME, {})
+            .get("value", {})
+        )
+        result_aspect = (
+            entry.get("aspects", {})
+            .get(DATAHUB_EXECUTION_REQUEST_RESULT_ASPECT_NAME, {})
+            .get("value", {})
+        )
+        key_aspect = (
+            entry.get("aspects", {})
+            .get(DATAHUB_EXECUTION_REQUEST_KEY_ASPECT_NAME, {})
+            .get("value", {})
+        )
+        return CleanupRecord(
+            urn=entry.get("urn"),
+            request_id=key_aspect.get("id"),
+            requested_at=input_aspect.get("requestedAt", 0),
+            status=result_aspect.get("status", "PENDING"),
+            ingestion_source=input_aspect.get("source", {}).get("ingestionSource", ""),
+        )
+
+    def _scroll_execution_requests(
+        self, overrides: Dict[str, Any] = {}
+    ) -> Iterator[CleanupRecord]:
+        headers: Dict[str, Any] = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+        params = {
+            "aspectNames": [
+                DATAHUB_EXECUTION_REQUEST_KEY_ASPECT_NAME,
+                DATAHUB_EXECUTION_REQUEST_INPUT_ASPECT_NAME,
+                DATAHUB_EXECUTION_REQUEST_RESULT_ASPECT_NAME,
+            ],
+            "count": str(self.config.batch_read_size),
+            "sort": "requestTimeMs",
+            "sortOrder": "DESCENDING",
+            "systemMetadata": "false",
+            "skipCache": "true",
+        }
+        params.update(overrides)
+
+        while True:
+            try:
+                url = f"{self.graph.config.server}/openapi/v2/entity/{DATAHUB_EXECUTION_REQUEST_ENTITY_NAME}"
+                response = self.graph._session.get(url, headers=headers, params=params)
+                response.raise_for_status()
+                document = response.json()
+
+                entries = document.get("results", [])
+                for entry in entries:
+                    yield self._to_cleanup_record(entry)
+
+                if "scrollId" not in document:
+                    break
+                params["scrollId"] = document["scrollId"]
+            except Exception as e:
+                logger.error(
+                    f"ergc({self.instance_id}): failed to fetch next batch of execution requests: {e}"
+                )
+                self.report.execution_request_cleanup_read_errors += 1
+
+    def _scroll_garbage_records(self):
+        state: Dict[str, Dict] = {}
+
+        now_ms = int(time.time()) * 1000
+        running_guard_timeout = now_ms - 30 * 24 * 3600 * 1000
+
+        for entry in self._scroll_execution_requests():
+            self.report.execution_request_cleanup_records_read += 1
+            key = entry.ingestion_source
+
+            # Always delete corrupted records
+            if not key:
+                logger.warning(
+                    f"ergc({self.instance_id}): will delete corrupted entry with missing source key: {entry}"
+                )
+                yield entry
+                continue
+
+            if key not in state:
+                state[key] = {}
+                state[key]["cutoffTimestamp"] = (
+                    entry.requested_at - self.config.keep_history_max_milliseconds()
+                )
+
+            state[key]["count"] = state[key].get("count", 0) + 1
+
+            # Do not delete if number of requests is below minimum
+            if state[key]["count"] < self.config.keep_history_min_count:
+                self.report.execution_request_cleanup_records_preserved += 1
+                continue
+
+            # Do not delete if number of requests do not exceed allowed maximum,
+            # or the cutoff date.
+            if (state[key]["count"] < self.config.keep_history_max_count) and (
+                entry.requested_at > state[key]["cutoffTimestamp"]
+            ):
+                self.report.execution_request_cleanup_records_preserved += 1
+                continue
+
+            # Do not delete if status is RUNNING or PENDING and created within last month. If the record is >month old and it did not
+            # transition to a final state within that timeframe, it likely has no value.
+            if entry.requested_at > running_guard_timeout and entry.status in [
+                "RUNNING",
+                "PENDING",
+            ]:
+                self.report.execution_request_cleanup_records_preserved += 1
+                continue
+
+            # Otherwise delete current record
+            logger.info(
+                (
+                    f"ergc({self.instance_id}): going to delete {entry.request_id} in source {key}; "
+                    f"source count: {state[key]['count']}; "
+                    f"source cutoff: {state[key]['cutoffTimestamp']}; "
+                    f"record timestamp: {entry.requested_at}."
+                )
+            )
+            self.report.execution_request_cleanup_records_deleted += 1
+            yield entry
+
+    def _delete_entry(self, entry: CleanupRecord) -> None:
+        try:
+            logger.info(
+                f"ergc({self.instance_id}): going to delete ExecutionRequest {entry.request_id}"
+            )
+            self.graph.delete_entity(entry.urn, True)
+        except Exception as e:
+            self.report.execution_request_cleanup_delete_errors += 1
+            logger.error(
+                f"ergc({self.instance_id}): failed to delete ExecutionRequest {entry.request_id}: {e}"
+            )
+
+    def run(self) -> None:
+        if not self.config.enabled:
+            logger.info(
+                f"ergc({self.instance_id}): ExecutionRequest cleaner is disabled."
+            )
+            return
+
+        logger.info(
+            (
+                f"ergc({self.instance_id}): Starting cleanup of ExecutionRequest records; "
+                f"max days: {self.config.keep_history_max_days}, "
+                f"min records: {self.config.keep_history_min_count}, "
+                f"max records: {self.config.keep_history_max_count}."
+            )
+        )
+
+        for entry in self._scroll_garbage_records():
+            self._delete_entry(entry)
+
+        logger.info(
+            f"ergc({self.instance_id}): Finished cleanup of ExecutionRequest records."
+        )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py
index 4da23c13659a7..93f004ab675ed 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py
@@ -20,6 +20,9 @@
 
 
 class SoftDeletedEntitiesCleanupConfig(ConfigModel):
+    enabled: bool = Field(
+        default=True, description="Whether to do soft deletion cleanup."
+    )
     retention_days: Optional[int] = Field(
         10,
         description="Number of days to retain metadata in DataHub",
@@ -59,6 +62,14 @@ class SoftDeletedEntitiesCleanupConfig(ConfigModel):
         default=None,
         description="Query to filter entities",
     )
+    limit_entities_delete: Optional[int] = Field(
+        25000, description="Max number of entities to delete."
+    )
+
+    runtime_limit_seconds: Optional[int] = Field(
+        None,
+        description="Runtime limit in seconds",
+    )
 
 
 @dataclass
@@ -96,7 +107,7 @@ def __init__(
     def delete_entity(self, urn: str) -> None:
         assert self.ctx.graph
 
-        entity_urn = Urn.create_from_string(urn)
+        entity_urn = Urn.from_string(urn)
         self.report.num_soft_deleted_entity_removed += 1
         self.report.num_soft_deleted_entity_removed_by_type[entity_urn.entity_type] = (
             self.report.num_soft_deleted_entity_removed_by_type.get(
@@ -122,6 +133,10 @@ def delete_entity(self, urn: str) -> None:
             return
 
         self.ctx.graph.delete_entity(urn=urn, hard=True)
+        self.ctx.graph.delete_references_to_urn(
+            urn=urn,
+            dry_run=False,
+        )
 
     def delete_soft_deleted_entity(self, urn: str) -> None:
         assert self.ctx.graph
@@ -144,7 +159,10 @@ def delete_soft_deleted_entity(self, urn: str) -> None:
                 self.delete_entity(urn)
 
     def cleanup_soft_deleted_entities(self) -> None:
+        if not self.config.enabled:
+            return
         assert self.ctx.graph
+        start_time = time.time()
 
         deleted_count_retention = 0
         urns = self.ctx.graph.get_urns_by_filter(
@@ -158,7 +176,26 @@ def cleanup_soft_deleted_entities(self) -> None:
 
         futures = {}
         with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor:
+            num_urns_submitted = 0
             for urn in urns:
+                num_urns_submitted += 1
+                if (
+                    self.config.limit_entities_delete
+                    and num_urns_submitted > self.config.limit_entities_delete
+                ):
+                    logger.info(
+                        f"Limit of {self.config.limit_entities_delete} entities reached. Stopping"
+                    )
+                    break
+                if (
+                    self.config.runtime_limit_seconds
+                    and time.time() - start_time > self.config.runtime_limit_seconds
+                ):
+                    logger.info(
+                        f"Runtime limit of {self.config.runtime_limit_seconds} seconds reached. Stopping"
+                    )
+                    break
+
                 future = executor.submit(self.delete_soft_deleted_entity, urn)
                 futures[future] = urn
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
index d175fce04a52c..c2c43a0e805b2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
@@ -7,6 +7,7 @@
 import functools
 import json
 import logging
+import re
 import threading
 import traceback
 import unittest.mock
@@ -55,8 +56,12 @@
     Cardinality,
     convert_to_cardinality,
 )
-from datahub.ingestion.source.sql.sql_common import SQLSourceReport
-from datahub.metadata.com.linkedin.pegasus2avro.schema import EditableSchemaMetadata
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
+from datahub.ingestion.source.sql.sql_types import resolve_sql_type
+from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+    EditableSchemaMetadata,
+    NumberType,
+)
 from datahub.metadata.schema_classes import (
     DatasetFieldProfileClass,
     DatasetProfileClass,
@@ -123,6 +128,8 @@
 
 _datasource_connection_injection_lock = threading.Lock()
 
+NORMALIZE_TYPE_PATTERN = re.compile(r"^(.*?)(?:[\[<(].*)?$")
+
 
 @contextlib.contextmanager
 def _inject_connection_into_datasource(conn: Connection) -> Iterator[None]:
@@ -165,11 +172,9 @@ def get_column_unique_count_dh_patch(self: SqlAlchemyDataset, column: str) -> in
         return convert_to_json_serializable(element_values.fetchone()[0])
     elif self.engine.dialect.name.lower() == BIGQUERY:
         element_values = self.engine.execute(
-            sa.select(
-                [
-                    sa.func.coalesce(sa.text(f"APPROX_COUNT_DISTINCT(`{column}`)")),
-                ]
-            ).select_from(self._table)
+            sa.select(sa.func.APPROX_COUNT_DISTINCT(sa.column(column))).select_from(
+                self._table
+            )
         )
         return convert_to_json_serializable(element_values.fetchone()[0])
     elif self.engine.dialect.name.lower() == SNOWFLAKE:
@@ -360,6 +365,8 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase):
     platform: str
     env: str
 
+    column_types: Dict[str, str] = dataclasses.field(default_factory=dict)
+
     def _get_columns_to_profile(self) -> List[str]:
         if not self.config.any_field_level_metrics_enabled():
             return []
@@ -373,11 +380,15 @@ def _get_columns_to_profile(self) -> List[str]:
 
         for col_dict in self.dataset.columns:
             col = col_dict["name"]
+            self.column_types[col] = str(col_dict["type"])
             # We expect the allow/deny patterns to specify '<table_pattern>.<column_pattern>'
             if not self.config._allow_deny_patterns.allowed(
                 f"{self.dataset_name}.{col}"
             ):
                 ignored_columns_by_pattern.append(col)
+            # We try to ignore nested columns as well
+            elif not self.config.profile_nested_fields and "." in col:
+                ignored_columns_by_pattern.append(col)
             elif col_dict.get("type") and self._should_ignore_column(col_dict["type"]):
                 ignored_columns_by_type.append(col)
             else:
@@ -407,9 +418,18 @@ def _get_columns_to_profile(self) -> List[str]:
         return columns_to_profile
 
     def _should_ignore_column(self, sqlalchemy_type: sa.types.TypeEngine) -> bool:
-        return str(sqlalchemy_type) in _get_column_types_to_ignore(
-            self.dataset.engine.dialect.name
-        )
+        # We don't profiles columns with None types
+        if str(sqlalchemy_type) == "NULL":
+            return True
+
+        sql_type = str(sqlalchemy_type)
+
+        match = re.match(NORMALIZE_TYPE_PATTERN, sql_type)
+
+        if match:
+            sql_type = match.group(1)
+
+        return sql_type in _get_column_types_to_ignore(self.dataset.engine.dialect.name)
 
     @_run_with_query_combiner
     def _get_column_type(self, column_spec: _SingleColumnSpec, column: str) -> None:
@@ -417,6 +437,21 @@ def _get_column_type(self, column_spec: _SingleColumnSpec, column: str) -> None:
             self.dataset, column
         )
 
+        if column_spec.type_ == ProfilerDataType.UNKNOWN:
+            try:
+                datahub_field_type = resolve_sql_type(
+                    self.column_types[column], self.dataset.engine.dialect.name.lower()
+                )
+            except Exception as e:
+                logger.debug(
+                    f"Error resolving sql type {self.column_types[column]}: {e}"
+                )
+                datahub_field_type = None
+            if datahub_field_type is None:
+                return
+            if isinstance(datahub_field_type, NumberType):
+                column_spec.type_ = ProfilerDataType.NUMERIC
+
     @_run_with_query_combiner
     def _get_column_cardinality(
         self, column_spec: _SingleColumnSpec, column: str
@@ -1397,6 +1432,8 @@ def _get_ge_dataset(
 def _get_column_types_to_ignore(dialect_name: str) -> List[str]:
     if dialect_name.lower() == POSTGRESQL:
         return ["JSON"]
+    elif dialect_name.lower() == BIGQUERY:
+        return ["ARRAY", "STRUCT", "GEOGRAPHY", "JSON"]
 
     return []
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py
index 2a9068d3d49d8..42d0def0a46e7 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py
@@ -19,7 +19,7 @@
 logger = logging.getLogger(__name__)
 
 
-class GEProfilingConfig(ConfigModel):
+class GEProfilingBaseConfig(ConfigModel):
     enabled: bool = Field(
         default=False, description="Whether profiling should be done."
     )
@@ -35,15 +35,6 @@ class GEProfilingConfig(ConfigModel):
         default=None,
         description="Offset in documents to profile. By default, uses no offset.",
     )
-    report_dropped_profiles: bool = Field(
-        default=False,
-        description="Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.",
-    )
-
-    turn_off_expensive_profiling_metrics: bool = Field(
-        default=False,
-        description="Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.",
-    )
     profile_table_level_only: bool = Field(
         default=False,
         description="Whether to perform profiling at table-level only, or include column-level profiling as well.",
@@ -92,6 +83,28 @@ class GEProfilingConfig(ConfigModel):
         default=True,
         description="Whether to profile for the sample values for all columns.",
     )
+
+    # The default of (5 * cpu_count) is adopted from the default max_workers
+    # parameter of ThreadPoolExecutor. Given that profiling is often an I/O-bound
+    # task, it may make sense to increase this default value in the future.
+    # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
+    max_workers: int = Field(
+        default=5 * (os.cpu_count() or 4),
+        description="Number of worker threads to use for profiling. Set to 1 to disable.",
+    )
+
+
+class GEProfilingConfig(GEProfilingBaseConfig):
+    report_dropped_profiles: bool = Field(
+        default=False,
+        description="Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.",
+    )
+
+    turn_off_expensive_profiling_metrics: bool = Field(
+        default=False,
+        description="Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.",
+    )
+
     field_sample_values_limit: int = Field(
         default=20,
         description="Upper limit for number of sample values to collect for all columns.",
@@ -112,12 +125,16 @@ class GEProfilingConfig(ConfigModel):
 
     profile_table_size_limit: Optional[int] = Field(
         default=5,
-        description="Profile tables only if their size is less then specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `snowflake` and `BigQuery`",
+        description="Profile tables only if their size is less than specified GBs. If set to `null`, "
+        "no limit on the size of tables to profile. Supported only in `snowflake` and `BigQuery`"
+        "Supported for `oracle` based on calculated size from gathered stats.",
     )
 
     profile_table_row_limit: Optional[int] = Field(
         default=5000000,
-        description="Profile tables only if their row count is less then specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `snowflake` and `BigQuery`",
+        description="Profile tables only if their row count is less than specified count. If set to `null`, "
+        "no limit on the row count of tables to profile. Supported only in `snowflake` and `BigQuery`"
+        "Supported for `oracle` based on gathered stats.",
     )
 
     profile_table_row_count_estimate_only: bool = Field(
@@ -126,15 +143,6 @@ class GEProfilingConfig(ConfigModel):
         "less accurate. Only supported for Postgres and MySQL. ",
     )
 
-    # The default of (5 * cpu_count) is adopted from the default max_workers
-    # parameter of ThreadPoolExecutor. Given that profiling is often an I/O-bound
-    # task, it may make sense to increase this default value in the future.
-    # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
-    max_workers: int = Field(
-        default=5 * (os.cpu_count() or 4),
-        description="Number of worker threads to use for profiling. Set to 1 to disable.",
-    )
-
     # The query combiner enables us to combine multiple queries into a single query,
     # reducing the number of round-trips to the database and speeding up profiling.
     query_combiner_enabled: bool = Field(
@@ -180,6 +188,11 @@ class GEProfilingConfig(ConfigModel):
         ),
     )
 
+    profile_nested_fields: bool = Field(
+        default=False,
+        description="Whether to profile complex types like structs, arrays and maps. ",
+    )
+
     @pydantic.root_validator(pre=True)
     def deprecate_bigquery_temp_table_schema(cls, values):
         # TODO: Update docs to remove mention of this field.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py
index b5caa83b2ff37..76f24bfd63d47 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py
@@ -1,10 +1,17 @@
 import json
 import logging
+import threading
 import uuid
 from typing import Any, Dict, Iterable, List, Optional
 
 from pyiceberg.catalog import Catalog
-from pyiceberg.exceptions import NoSuchIcebergTableError
+from pyiceberg.exceptions import (
+    NoSuchIcebergTableError,
+    NoSuchNamespaceError,
+    NoSuchPropertyException,
+    NoSuchTableError,
+    ServerError,
+)
 from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType, visit
 from pyiceberg.table import Table
 from pyiceberg.typedef import Identifier
@@ -75,6 +82,8 @@
     OwnershipClass,
     OwnershipTypeClass,
 )
+from datahub.utilities.perf_timer import PerfTimer
+from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor
 
 LOGGER = logging.getLogger(__name__)
 logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
@@ -97,7 +106,7 @@
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default.")
 @capability(
     SourceCapability.OWNERSHIP,
-    "Optionally enabled via configuration by specifying which Iceberg table property holds user or group ownership.",
+    "Automatically ingests ownership information from table properties based on `user_ownership_property` and `group_ownership_property`",
 )
 @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
 class IcebergSource(StatefulIngestionSourceBase):
@@ -130,73 +139,180 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         ]
 
     def _get_datasets(self, catalog: Catalog) -> Iterable[Identifier]:
-        for namespace in catalog.list_namespaces():
-            yield from catalog.list_tables(namespace)
+        namespaces = catalog.list_namespaces()
+        LOGGER.debug(
+            f"Retrieved {len(namespaces)} namespaces, first 10: {namespaces[:10]}"
+        )
+        self.report.report_no_listed_namespaces(len(namespaces))
+        tables_count = 0
+        for namespace in namespaces:
+            namespace_repr = ".".join(namespace)
+            if not self.config.namespace_pattern.allowed(namespace_repr):
+                LOGGER.info(
+                    f"Namespace {namespace_repr} is not allowed by config pattern, skipping"
+                )
+                self.report.report_dropped(f"{namespace_repr}.*")
+                continue
+            try:
+                tables = catalog.list_tables(namespace)
+                tables_count += len(tables)
+                LOGGER.debug(
+                    f"Retrieved {len(tables)} tables for namespace: {namespace}, in total retrieved {tables_count}, first 10: {tables[:10]}"
+                )
+                self.report.report_listed_tables_for_namespace(
+                    ".".join(namespace), len(tables)
+                )
+                yield from tables
+            except NoSuchNamespaceError:
+                self.report.report_warning(
+                    "no-such-namespace",
+                    f"Couldn't list tables for namespace {namespace} due to NoSuchNamespaceError exception",
+                )
+                LOGGER.warning(
+                    f"NoSuchNamespaceError exception while trying to get list of tables from namespace {namespace}, skipping it",
+                )
+            except Exception as e:
+                self.report.report_failure(
+                    "listing-tables-exception",
+                    f"Couldn't list tables for namespace {namespace} due to {e}",
+                )
+                LOGGER.exception(
+                    f"Unexpected exception while trying to get list of tables for namespace {namespace}, skipping it"
+                )
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
-        try:
-            catalog = self.config.get_catalog()
-        except Exception as e:
-            LOGGER.error("Failed to get catalog", exc_info=True)
-            self.report.report_failure("get-catalog", f"Failed to get catalog: {e}")
-            return
+        thread_local = threading.local()
 
-        for dataset_path in self._get_datasets(catalog):
+        def _process_dataset(dataset_path: Identifier) -> Iterable[MetadataWorkUnit]:
+            LOGGER.debug(f"Processing dataset for path {dataset_path}")
             dataset_name = ".".join(dataset_path)
             if not self.config.table_pattern.allowed(dataset_name):
                 # Dataset name is rejected by pattern, report as dropped.
                 self.report.report_dropped(dataset_name)
-                continue
-
+                LOGGER.debug(
+                    f"Skipping table {dataset_name} due to not being allowed by the config pattern"
+                )
+                return
             try:
-                # Try to load an Iceberg table.  Might not contain one, this will be caught by NoSuchIcebergTableError.
-                table = catalog.load_table(dataset_path)
+                if not hasattr(thread_local, "local_catalog"):
+                    LOGGER.debug(
+                        f"Didn't find local_catalog in thread_local ({thread_local}), initializing new catalog"
+                    )
+                    thread_local.local_catalog = self.config.get_catalog()
+
+                with PerfTimer() as timer:
+                    table = thread_local.local_catalog.load_table(dataset_path)
+                    time_taken = timer.elapsed_seconds()
+                    self.report.report_table_load_time(time_taken)
+                LOGGER.debug(f"Loaded table: {table.name()}, time taken: {time_taken}")
                 yield from self._create_iceberg_workunit(dataset_name, table)
+            except NoSuchPropertyException as e:
+                self.report.report_warning(
+                    "table-property-missing",
+                    f"Failed to create workunit for {dataset_name}. {e}",
+                )
+                LOGGER.warning(
+                    f"NoSuchPropertyException while processing table {dataset_path}, skipping it.",
+                )
             except NoSuchIcebergTableError as e:
+                self.report.report_warning(
+                    "not-an-iceberg-table",
+                    f"Failed to create workunit for {dataset_name}. {e}",
+                )
+                LOGGER.warning(
+                    f"NoSuchIcebergTableError while processing table {dataset_path}, skipping it.",
+                )
+            except NoSuchTableError as e:
+                self.report.report_warning(
+                    "no-such-table",
+                    f"Failed to create workunit for {dataset_name}. {e}",
+                )
+                LOGGER.warning(
+                    f"NoSuchTableError while processing table {dataset_path}, skipping it.",
+                )
+            except FileNotFoundError as e:
+                self.report.report_warning(
+                    "file-not-found",
+                    f"Encountered FileNotFoundError when trying to read manifest file for {dataset_name}. {e}",
+                )
+                LOGGER.warning(
+                    f"FileNotFoundError while processing table {dataset_path}, skipping it."
+                )
+            except ServerError as e:
+                self.report.report_warning(
+                    "iceberg-rest-server-error",
+                    f"Iceberg Rest Catalog returned 500 status due to an unhandled exception for {dataset_name}. Exception: {e}",
+                )
+                LOGGER.warning(
+                    f"Iceberg Rest Catalog server error (500 status) encountered when processing table {dataset_path}, skipping it."
+                )
+            except Exception as e:
                 self.report.report_failure("general", f"Failed to create workunit: {e}")
                 LOGGER.exception(
                     f"Exception while processing table {dataset_path}, skipping it.",
                 )
 
+        try:
+            catalog = self.config.get_catalog()
+        except Exception as e:
+            self.report.report_failure("get-catalog", f"Failed to get catalog: {e}")
+            return
+
+        for wu in ThreadedIteratorExecutor.process(
+            worker_func=_process_dataset,
+            args_list=[(dataset_path,) for dataset_path in self._get_datasets(catalog)],
+            max_workers=self.config.processing_threads,
+        ):
+            yield wu
+
     def _create_iceberg_workunit(
         self, dataset_name: str, table: Table
     ) -> Iterable[MetadataWorkUnit]:
-        self.report.report_table_scanned(dataset_name)
-        dataset_urn: str = make_dataset_urn_with_platform_instance(
-            self.platform,
-            dataset_name,
-            self.config.platform_instance,
-            self.config.env,
-        )
-        dataset_snapshot = DatasetSnapshot(
-            urn=dataset_urn,
-            aspects=[Status(removed=False)],
-        )
-
-        # Dataset properties aspect.
-        custom_properties = table.metadata.properties.copy()
-        custom_properties["location"] = table.metadata.location
-        custom_properties["format-version"] = str(table.metadata.format_version)
-        custom_properties["partition-spec"] = str(self._get_partition_aspect(table))
-        if table.current_snapshot():
-            custom_properties["snapshot-id"] = str(table.current_snapshot().snapshot_id)
-            custom_properties["manifest-list"] = table.current_snapshot().manifest_list
-        dataset_properties = DatasetPropertiesClass(
-            tags=[],
-            description=table.metadata.properties.get("comment", None),
-            customProperties=custom_properties,
-        )
-        dataset_snapshot.aspects.append(dataset_properties)
+        with PerfTimer() as timer:
+            self.report.report_table_scanned(dataset_name)
+            LOGGER.debug(f"Processing table {dataset_name}")
+            dataset_urn: str = make_dataset_urn_with_platform_instance(
+                self.platform,
+                dataset_name,
+                self.config.platform_instance,
+                self.config.env,
+            )
+            dataset_snapshot = DatasetSnapshot(
+                urn=dataset_urn,
+                aspects=[Status(removed=False)],
+            )
 
-        # Dataset ownership aspect.
-        dataset_ownership = self._get_ownership_aspect(table)
-        if dataset_ownership:
-            dataset_snapshot.aspects.append(dataset_ownership)
+            # Dataset properties aspect.
+            custom_properties = table.metadata.properties.copy()
+            custom_properties["location"] = table.metadata.location
+            custom_properties["format-version"] = str(table.metadata.format_version)
+            custom_properties["partition-spec"] = str(self._get_partition_aspect(table))
+            if table.current_snapshot():
+                custom_properties["snapshot-id"] = str(
+                    table.current_snapshot().snapshot_id
+                )
+                custom_properties[
+                    "manifest-list"
+                ] = table.current_snapshot().manifest_list
+            dataset_properties = DatasetPropertiesClass(
+                name=table.name()[-1],
+                description=table.metadata.properties.get("comment", None),
+                customProperties=custom_properties,
+            )
+            dataset_snapshot.aspects.append(dataset_properties)
+            # Dataset ownership aspect.
+            dataset_ownership = self._get_ownership_aspect(table)
+            if dataset_ownership:
+                LOGGER.debug(
+                    f"Adding ownership: {dataset_ownership} to the dataset {dataset_name}"
+                )
+                dataset_snapshot.aspects.append(dataset_ownership)
 
-        schema_metadata = self._create_schema_metadata(dataset_name, table)
-        dataset_snapshot.aspects.append(schema_metadata)
+            schema_metadata = self._create_schema_metadata(dataset_name, table)
+            dataset_snapshot.aspects.append(schema_metadata)
 
-        mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
+            mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
+        self.report.report_table_processing_time(timer.elapsed_seconds())
         yield MetadataWorkUnit(id=dataset_name, mce=mce)
 
         dpi_aspect = self._get_dataplatform_instance_aspect(dataset_urn=dataset_urn)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py
index b74c096d0798e..4a7f6bf4d60c1 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py
@@ -2,6 +2,7 @@
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional
 
+from humanfriendly import format_timespan
 from pydantic import Field, validator
 from pyiceberg.catalog import Catalog, load_catalog
 
@@ -18,6 +19,7 @@
     OperationConfig,
     is_profiling_enabled,
 )
+from datahub.utilities.stats_collections import TopKDict, int_top_k_dict
 
 logger = logging.getLogger(__name__)
 
@@ -66,6 +68,10 @@ class IcebergSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin)
         default=AllowDenyPattern.allow_all(),
         description="Regex patterns for tables to filter in ingestion.",
     )
+    namespace_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for namespaces to filter in ingestion.",
+    )
     user_ownership_property: Optional[str] = Field(
         default="owner",
         description="Iceberg table property to look for a `CorpUser` owner.  Can only hold a single user value.  If property has no value, no owner information will be emitted.",
@@ -75,6 +81,9 @@ class IcebergSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin)
         description="Iceberg table property to look for a `CorpGroup` owner.  Can only hold a single group value.  If property has no value, no owner information will be emitted.",
     )
     profiling: IcebergProfilingConfig = IcebergProfilingConfig()
+    processing_threads: int = Field(
+        default=1, description="How many threads will be processing tables"
+    )
 
     @validator("catalog", pre=True, always=True)
     def handle_deprecated_catalog_format(cls, value):
@@ -131,17 +140,72 @@ def get_catalog(self) -> Catalog:
 
         # Retrieve the dict associated with the one catalog entry
         catalog_name, catalog_config = next(iter(self.catalog.items()))
+        logger.debug(
+            "Initializing the catalog %s with config: %s", catalog_name, catalog_config
+        )
         return load_catalog(name=catalog_name, **catalog_config)
 
 
+class TimingClass:
+    times: List[int]
+
+    def __init__(self):
+        self.times = []
+
+    def add_timing(self, t):
+        self.times.append(t)
+
+    def __str__(self):
+        if len(self.times) == 0:
+            return "no timings reported"
+        self.times.sort()
+        total = sum(self.times)
+        avg = total / len(self.times)
+        return str(
+            {
+                "average_time": format_timespan(avg, detailed=True, max_units=3),
+                "min_time": format_timespan(self.times[0], detailed=True, max_units=3),
+                "max_time": format_timespan(self.times[-1], detailed=True, max_units=3),
+                # total_time does not provide correct information in case we run in more than 1 thread
+                "total_time": format_timespan(total, detailed=True, max_units=3),
+            }
+        )
+
+
 @dataclass
 class IcebergSourceReport(StaleEntityRemovalSourceReport):
     tables_scanned: int = 0
     entities_profiled: int = 0
     filtered: List[str] = field(default_factory=list)
+    load_table_timings: TimingClass = field(default_factory=TimingClass)
+    processing_table_timings: TimingClass = field(default_factory=TimingClass)
+    profiling_table_timings: TimingClass = field(default_factory=TimingClass)
+    listed_namespaces: int = 0
+    total_listed_tables: int = 0
+    tables_listed_per_namespace: TopKDict[str, int] = field(
+        default_factory=int_top_k_dict
+    )
+
+    def report_listed_tables_for_namespace(
+        self, namespace: str, no_tables: int
+    ) -> None:
+        self.tables_listed_per_namespace[namespace] = no_tables
+        self.total_listed_tables += no_tables
+
+    def report_no_listed_namespaces(self, amount: int) -> None:
+        self.listed_namespaces = amount
 
     def report_table_scanned(self, name: str) -> None:
         self.tables_scanned += 1
 
     def report_dropped(self, ent_name: str) -> None:
         self.filtered.append(ent_name)
+
+    def report_table_load_time(self, t: float) -> None:
+        self.load_table_timings.add_timing(t)
+
+    def report_table_processing_time(self, t: float) -> None:
+        self.processing_table_timings.add_timing(t)
+
+    def report_table_profiling_time(self, t: float) -> None:
+        self.profiling_table_timings.add_timing(t)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_profiler.py
index e1d52752d779a..9cc6dd08544e4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_profiler.py
@@ -1,3 +1,4 @@
+import logging
 from typing import Any, Callable, Dict, Iterable, Union, cast
 
 from pyiceberg.conversions import from_bytes
@@ -33,6 +34,9 @@
     DatasetFieldProfileClass,
     DatasetProfileClass,
 )
+from datahub.utilities.perf_timer import PerfTimer
+
+LOGGER = logging.getLogger(__name__)
 
 
 class IcebergProfiler:
@@ -109,92 +113,101 @@ def profile_table(
         Yields:
             Iterator[Iterable[MetadataWorkUnit]]: Workunits related to datasetProfile.
         """
-        current_snapshot = table.current_snapshot()
-        if not current_snapshot:
-            # Table has no data, cannot profile, or we can't get current_snapshot.
-            return
-
-        row_count = (
-            int(current_snapshot.summary.additional_properties["total-records"])
-            if current_snapshot.summary
-            else 0
-        )
-        column_count = len(
-            [
-                field.field_id
-                for field in table.schema().fields
-                if field.field_type.is_primitive
-            ]
-        )
-        dataset_profile = DatasetProfileClass(
-            timestampMillis=get_sys_time(),
-            rowCount=row_count,
-            columnCount=column_count,
-        )
-        dataset_profile.fieldProfiles = []
+        with PerfTimer() as timer:
+            LOGGER.debug(f"Starting profiling of dataset: {dataset_name}")
+            current_snapshot = table.current_snapshot()
+            if not current_snapshot:
+                # Table has no data, cannot profile, or we can't get current_snapshot.
+                return
+
+            row_count = (
+                int(current_snapshot.summary.additional_properties["total-records"])
+                if current_snapshot.summary
+                else 0
+            )
+            column_count = len(
+                [
+                    field.field_id
+                    for field in table.schema().fields
+                    if field.field_type.is_primitive
+                ]
+            )
+            dataset_profile = DatasetProfileClass(
+                timestampMillis=get_sys_time(),
+                rowCount=row_count,
+                columnCount=column_count,
+            )
+            dataset_profile.fieldProfiles = []
 
-        total_count = 0
-        null_counts: Dict[int, int] = {}
-        min_bounds: Dict[int, Any] = {}
-        max_bounds: Dict[int, Any] = {}
-        try:
-            for manifest in current_snapshot.manifests(table.io):
-                for manifest_entry in manifest.fetch_manifest_entry(table.io):
-                    data_file = manifest_entry.data_file
+            total_count = 0
+            null_counts: Dict[int, int] = {}
+            min_bounds: Dict[int, Any] = {}
+            max_bounds: Dict[int, Any] = {}
+            try:
+                for manifest in current_snapshot.manifests(table.io):
+                    for manifest_entry in manifest.fetch_manifest_entry(table.io):
+                        data_file = manifest_entry.data_file
+                        if self.config.include_field_null_count:
+                            null_counts = self._aggregate_counts(
+                                null_counts, data_file.null_value_counts
+                            )
+                        if self.config.include_field_min_value:
+                            self._aggregate_bounds(
+                                table.schema(),
+                                min,
+                                min_bounds,
+                                data_file.lower_bounds,
+                            )
+                        if self.config.include_field_max_value:
+                            self._aggregate_bounds(
+                                table.schema(),
+                                max,
+                                max_bounds,
+                                data_file.upper_bounds,
+                            )
+                        total_count += data_file.record_count
+            except Exception as e:
+                # Catch any errors that arise from attempting to read the Iceberg table's manifests
+                # This will prevent stateful ingestion from being blocked by an error (profiling is not critical)
+                self.report.report_warning(
+                    "profiling",
+                    f"Error while profiling dataset {dataset_name}: {e}",
+                )
+            if row_count:
+                # Iterating through fieldPaths introduces unwanted stats for list element fields...
+                for field_path, field_id in table.schema()._name_to_id.items():
+                    field = table.schema().find_field(field_id)
+                    column_profile = DatasetFieldProfileClass(fieldPath=field_path)
                     if self.config.include_field_null_count:
-                        null_counts = self._aggregate_counts(
-                            null_counts, data_file.null_value_counts
+                        column_profile.nullCount = cast(
+                            int, null_counts.get(field_id, 0)
+                        )
+                        column_profile.nullProportion = float(
+                            column_profile.nullCount / row_count
                         )
+
                     if self.config.include_field_min_value:
-                        self._aggregate_bounds(
-                            table.schema(),
-                            min,
-                            min_bounds,
-                            data_file.lower_bounds,
+                        column_profile.min = (
+                            self._render_value(
+                                dataset_name, field.field_type, min_bounds.get(field_id)
+                            )
+                            if field_id in min_bounds
+                            else None
                         )
                     if self.config.include_field_max_value:
-                        self._aggregate_bounds(
-                            table.schema(),
-                            max,
-                            max_bounds,
-                            data_file.upper_bounds,
+                        column_profile.max = (
+                            self._render_value(
+                                dataset_name, field.field_type, max_bounds.get(field_id)
+                            )
+                            if field_id in max_bounds
+                            else None
                         )
-                    total_count += data_file.record_count
-        except Exception as e:
-            # Catch any errors that arise from attempting to read the Iceberg table's manifests
-            # This will prevent stateful ingestion from being blocked by an error (profiling is not critical)
-            self.report.report_warning(
-                "profiling",
-                f"Error while profiling dataset {dataset_name}: {e}",
+                    dataset_profile.fieldProfiles.append(column_profile)
+            time_taken = timer.elapsed_seconds()
+            self.report.report_table_profiling_time(time_taken)
+            LOGGER.debug(
+                f"Finished profiling of dataset: {dataset_name} in {time_taken}"
             )
-        if row_count:
-            # Iterating through fieldPaths introduces unwanted stats for list element fields...
-            for field_path, field_id in table.schema()._name_to_id.items():
-                field = table.schema().find_field(field_id)
-                column_profile = DatasetFieldProfileClass(fieldPath=field_path)
-                if self.config.include_field_null_count:
-                    column_profile.nullCount = cast(int, null_counts.get(field_id, 0))
-                    column_profile.nullProportion = float(
-                        column_profile.nullCount / row_count
-                    )
-
-                if self.config.include_field_min_value:
-                    column_profile.min = (
-                        self._render_value(
-                            dataset_name, field.field_type, min_bounds.get(field_id)
-                        )
-                        if field_id in min_bounds
-                        else None
-                    )
-                if self.config.include_field_max_value:
-                    column_profile.max = (
-                        self._render_value(
-                            dataset_name, field.field_type, max_bounds.get(field_id)
-                        )
-                        if field_id in max_bounds
-                        else None
-                    )
-                dataset_profile.fieldProfiles.append(column_profile)
 
         yield MetadataChangeProposalWrapper(
             entityUrn=dataset_urn,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py
similarity index 90%
rename from metadata-ingestion/src/datahub/ingestion/source/kafka.py
rename to metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py
index a757250a0d6c8..fa842a15ba732 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py
@@ -18,6 +18,7 @@
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.kafka import KafkaConsumerConnectionConfig
+from datahub.configuration.kafka_consumer_config import CallableConsumerConfig
 from datahub.configuration.source_common import (
     DatasetSourceConfigMixin,
     LowerCaseDatasetUrnConfigMixin,
@@ -49,7 +50,9 @@
 )
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import DatasetSubTypes
-from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase
+from datahub.ingestion.source.kafka.kafka_schema_registry_base import (
+    KafkaSchemaRegistryBase,
+)
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
     StaleEntityRemovalSourceReport,
@@ -138,18 +141,50 @@ class KafkaSourceConfig(
         default=False,
         description="Disables the utilization of the TopicRecordNameStrategy for Schema Registry subjects. For more information, visit: https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#handling-differences-between-preregistered-and-client-derived-schemas:~:text=io.confluent.kafka.serializers.subject.TopicRecordNameStrategy",
     )
+    ingest_schemas_as_entities: bool = pydantic.Field(
+        default=False,
+        description="Enables ingesting schemas from schema registry as separate entities, in addition to the topics",
+    )
 
 
 def get_kafka_consumer(
     connection: KafkaConsumerConnectionConfig,
 ) -> confluent_kafka.Consumer:
-    return confluent_kafka.Consumer(
+    consumer = confluent_kafka.Consumer(
+        {
+            "group.id": "datahub-kafka-ingestion",
+            "bootstrap.servers": connection.bootstrap,
+            **connection.consumer_config,
+        }
+    )
+
+    if CallableConsumerConfig.is_callable_config(connection.consumer_config):
+        # As per documentation, we need to explicitly call the poll method to make sure OAuth callback gets executed
+        # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration
+        logger.debug("Initiating polling for kafka consumer")
+        consumer.poll(timeout=30)
+        logger.debug("Initiated polling for kafka consumer")
+
+    return consumer
+
+
+def get_kafka_admin_client(
+    connection: KafkaConsumerConnectionConfig,
+) -> AdminClient:
+    client = AdminClient(
         {
-            "group.id": "test",
+            "group.id": "datahub-kafka-ingestion",
             "bootstrap.servers": connection.bootstrap,
             **connection.consumer_config,
         }
     )
+    if CallableConsumerConfig.is_callable_config(connection.consumer_config):
+        # As per documentation, we need to explicitly call the poll method to make sure OAuth callback gets executed
+        # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration
+        logger.debug("Initiating polling for kafka admin client")
+        client.poll(timeout=30)
+        logger.debug("Initiated polling for kafka admin client")
+    return client
 
 
 @dataclass
@@ -266,13 +301,7 @@ def __init__(self, config: KafkaSourceConfig, ctx: PipelineContext):
     def init_kafka_admin_client(self) -> None:
         try:
             # TODO: Do we require separate config than existing consumer_config ?
-            self.admin_client = AdminClient(
-                {
-                    "group.id": "test",
-                    "bootstrap.servers": self.source_config.connection.bootstrap,
-                    **self.source_config.connection.consumer_config,
-                }
-            )
+            self.admin_client = get_kafka_admin_client(self.source_config.connection)
         except Exception as e:
             logger.debug(e, exc_info=e)
             self.report.report_warning(
@@ -318,17 +347,20 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             else:
                 self.report.report_dropped(topic)
 
-        # Get all subjects from schema registry and ingest them as SCHEMA DatasetSubTypes
-        for subject in self.schema_registry_client.get_subjects():
-            try:
-                yield from self._extract_record(
-                    subject, True, topic_detail=None, extra_topic_config=None
-                )
-            except Exception as e:
-                logger.warning(f"Failed to extract subject {subject}", exc_info=True)
-                self.report.report_warning(
-                    "subject", f"Exception while extracting topic {subject}: {e}"
-                )
+        if self.source_config.ingest_schemas_as_entities:
+            # Get all subjects from schema registry and ingest them as SCHEMA DatasetSubTypes
+            for subject in self.schema_registry_client.get_subjects():
+                try:
+                    yield from self._extract_record(
+                        subject, True, topic_detail=None, extra_topic_config=None
+                    )
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to extract subject {subject}", exc_info=True
+                    )
+                    self.report.report_warning(
+                        "subject", f"Exception while extracting topic {subject}: {e}"
+                    )
 
     def _extract_record(
         self,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_schema_registry_base.py
similarity index 100%
rename from metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py
rename to metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_schema_registry_base.py
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py
deleted file mode 100644
index 0b201278142e3..0000000000000
--- a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py
+++ /dev/null
@@ -1,1438 +0,0 @@
-import logging
-import re
-from dataclasses import dataclass, field
-from typing import Dict, Iterable, List, Optional, Tuple
-
-import jpype
-import jpype.imports
-import requests
-from pydantic.fields import Field
-from sqlalchemy.engine.url import make_url
-
-import datahub.emitter.mce_builder as builder
-import datahub.metadata.schema_classes as models
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
-from datahub.configuration.source_common import (
-    DatasetLineageProviderConfigBase,
-    PlatformInstanceConfigMixin,
-)
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.api.decorators import (
-    SourceCapability,
-    SupportStatus,
-    capability,
-    config_class,
-    platform_name,
-    support_status,
-)
-from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source
-from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.sql.sqlalchemy_uri_mapper import (
-    get_platform_from_sqlalchemy_uri,
-)
-from datahub.ingestion.source.state.stale_entity_removal_handler import (
-    StaleEntityRemovalHandler,
-    StaleEntityRemovalSourceReport,
-    StatefulStaleMetadataRemovalConfig,
-)
-from datahub.ingestion.source.state.stateful_ingestion_base import (
-    StatefulIngestionConfigBase,
-    StatefulIngestionSourceBase,
-)
-
-logger = logging.getLogger(__name__)
-
-KAFKA = "kafka"
-SOURCE = "source"
-SINK = "sink"
-CONNECTOR_CLASS = "connector.class"
-
-
-class ProvidedConfig(ConfigModel):
-    provider: str
-    path_key: str
-    value: str
-
-
-class GenericConnectorConfig(ConfigModel):
-    connector_name: str
-    source_dataset: str
-    source_platform: str
-
-
-class KafkaConnectSourceConfig(
-    PlatformInstanceConfigMixin,
-    DatasetLineageProviderConfigBase,
-    StatefulIngestionConfigBase,
-):
-    # See the Connect REST Interface for details
-    # https://docs.confluent.io/platform/current/connect/references/restapi.html#
-    connect_uri: str = Field(
-        default="http://localhost:8083/", description="URI to connect to."
-    )
-    username: Optional[str] = Field(default=None, description="Kafka Connect username.")
-    password: Optional[str] = Field(default=None, description="Kafka Connect password.")
-    cluster_name: Optional[str] = Field(
-        default="connect-cluster", description="Cluster to ingest from."
-    )
-    # convert lineage dataset's urns to lowercase
-    convert_lineage_urns_to_lowercase: bool = Field(
-        default=False,
-        description="Whether to convert the urns of ingested lineage dataset to lowercase",
-    )
-    connector_patterns: AllowDenyPattern = Field(
-        default=AllowDenyPattern.allow_all(),
-        description="regex patterns for connectors to filter for ingestion.",
-    )
-    provided_configs: Optional[List[ProvidedConfig]] = Field(
-        default=None, description="Provided Configurations"
-    )
-    connect_to_platform_map: Optional[Dict[str, Dict[str, str]]] = Field(
-        default=None,
-        description='Platform instance mapping when multiple instances for a platform is available. Entry for a platform should be in either `platform_instance_map` or `connect_to_platform_map`. e.g.`connect_to_platform_map: { "postgres-connector-finance-db": "postgres": "core_finance_instance" }`',
-    )
-    platform_instance_map: Optional[Dict[str, str]] = Field(
-        default=None,
-        description='Platform instance mapping to use when constructing URNs. e.g.`platform_instance_map: { "hive": "warehouse" }`',
-    )
-    generic_connectors: List[GenericConnectorConfig] = Field(
-        default=[],
-        description="Provide lineage graph for sources connectors other than Confluent JDBC Source Connector, Debezium Source Connector, and Mongo Source Connector",
-    )
-
-    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
-
-
-@dataclass
-class KafkaConnectSourceReport(StaleEntityRemovalSourceReport):
-    connectors_scanned: int = 0
-    filtered: List[str] = field(default_factory=list)
-
-    def report_connector_scanned(self, connector: str) -> None:
-        self.connectors_scanned += 1
-
-    def report_dropped(self, connector: str) -> None:
-        self.filtered.append(connector)
-
-
-@dataclass
-class KafkaConnectLineage:
-    """Class to store Kafka Connect lineage mapping, Each instance is potential DataJob"""
-
-    source_platform: str
-    target_dataset: str
-    target_platform: str
-    job_property_bag: Optional[Dict[str, str]] = None
-    source_dataset: Optional[str] = None
-
-
-@dataclass
-class ConnectorManifest:
-    """Each instance is potential DataFlow"""
-
-    name: str
-    type: str
-    config: Dict
-    tasks: Dict
-    url: Optional[str] = None
-    flow_property_bag: Optional[Dict[str, str]] = None
-    lineages: List[KafkaConnectLineage] = field(default_factory=list)
-    topic_names: Iterable[str] = field(default_factory=list)
-
-
-def remove_prefix(text: str, prefix: str) -> str:
-    if text.startswith(prefix):
-        index = len(prefix)
-        return text[index:]
-    return text
-
-
-def unquote(
-    string: str, leading_quote: str = '"', trailing_quote: Optional[str] = None
-) -> str:
-    """
-    If string starts and ends with a quote, unquote it
-    """
-    trailing_quote = trailing_quote if trailing_quote else leading_quote
-    if string.startswith(leading_quote) and string.endswith(trailing_quote):
-        string = string[1:-1]
-    return string
-
-
-def get_dataset_name(
-    database_name: Optional[str],
-    source_table: str,
-) -> str:
-    if database_name:
-        dataset_name = database_name + "." + source_table
-    else:
-        dataset_name = source_table
-
-    return dataset_name
-
-
-def get_platform_instance(
-    config: KafkaConnectSourceConfig, connector_name: str, platform: str
-) -> Optional[str]:
-    instance_name = None
-    if (
-        config.connect_to_platform_map
-        and config.connect_to_platform_map.get(connector_name)
-        and config.connect_to_platform_map[connector_name].get(platform)
-    ):
-        instance_name = config.connect_to_platform_map[connector_name][platform]
-        if config.platform_instance_map and config.platform_instance_map.get(platform):
-            logger.warning(
-                f"Same source platform {platform} configured in both platform_instance_map and connect_to_platform_map."
-                "Will prefer connector specific platform instance from connect_to_platform_map."
-            )
-    elif config.platform_instance_map and config.platform_instance_map.get(platform):
-        instance_name = config.platform_instance_map[platform]
-    logger.info(
-        f"Instance name assigned is: {instance_name} for Connector Name {connector_name} and platform {platform}"
-    )
-    return instance_name
-
-
-@dataclass
-class ConfluentJDBCSourceConnector:
-    connector_manifest: ConnectorManifest
-    report: KafkaConnectSourceReport
-
-    def __init__(
-        self,
-        connector_manifest: ConnectorManifest,
-        config: KafkaConnectSourceConfig,
-        report: KafkaConnectSourceReport,
-    ) -> None:
-        self.connector_manifest = connector_manifest
-        self.config = config
-        self.report = report
-        self._extract_lineages()
-
-    REGEXROUTER = "org.apache.kafka.connect.transforms.RegexRouter"
-    KNOWN_TOPICROUTING_TRANSFORMS = [REGEXROUTER]
-    # https://kafka.apache.org/documentation/#connect_included_transformation
-    KAFKA_NONTOPICROUTING_TRANSFORMS = [
-        "InsertField",
-        "InsertField$Key",
-        "InsertField$Value",
-        "ReplaceField",
-        "ReplaceField$Key",
-        "ReplaceField$Value",
-        "MaskField",
-        "MaskField$Key",
-        "MaskField$Value",
-        "ValueToKey",
-        "ValueToKey$Key",
-        "ValueToKey$Value",
-        "HoistField",
-        "HoistField$Key",
-        "HoistField$Value",
-        "ExtractField",
-        "ExtractField$Key",
-        "ExtractField$Value",
-        "SetSchemaMetadata",
-        "SetSchemaMetadata$Key",
-        "SetSchemaMetadata$Value",
-        "Flatten",
-        "Flatten$Key",
-        "Flatten$Value",
-        "Cast",
-        "Cast$Key",
-        "Cast$Value",
-        "HeadersFrom",
-        "HeadersFrom$Key",
-        "HeadersFrom$Value",
-        "TimestampConverter",
-        "Filter",
-        "InsertHeader",
-        "DropHeaders",
-    ]
-    # https://docs.confluent.io/platform/current/connect/transforms/overview.html
-    CONFLUENT_NONTOPICROUTING_TRANSFORMS = [
-        "Drop",
-        "Drop$Key",
-        "Drop$Value",
-        "Filter",
-        "Filter$Key",
-        "Filter$Value",
-        "TombstoneHandler",
-    ]
-    KNOWN_NONTOPICROUTING_TRANSFORMS = (
-        KAFKA_NONTOPICROUTING_TRANSFORMS
-        + [
-            f"org.apache.kafka.connect.transforms.{t}"
-            for t in KAFKA_NONTOPICROUTING_TRANSFORMS
-        ]
-        + CONFLUENT_NONTOPICROUTING_TRANSFORMS
-        + [
-            f"io.confluent.connect.transforms.{t}"
-            for t in CONFLUENT_NONTOPICROUTING_TRANSFORMS
-        ]
-    )
-
-    @dataclass
-    class JdbcParser:
-        db_connection_url: str
-        source_platform: str
-        database_name: str
-        topic_prefix: str
-        query: str
-        transforms: list
-
-    def report_warning(self, key: str, reason: str) -> None:
-        logger.warning(f"{key}: {reason}")
-        self.report.report_warning(key, reason)
-
-    def get_parser(
-        self,
-        connector_manifest: ConnectorManifest,
-    ) -> JdbcParser:
-        url = remove_prefix(
-            str(connector_manifest.config.get("connection.url")), "jdbc:"
-        )
-        url_instance = make_url(url)
-        source_platform = get_platform_from_sqlalchemy_uri(str(url_instance))
-        database_name = url_instance.database
-        assert database_name
-        db_connection_url = f"{url_instance.drivername}://{url_instance.host}:{url_instance.port}/{database_name}"
-
-        topic_prefix = self.connector_manifest.config.get("topic.prefix", None)
-
-        query = self.connector_manifest.config.get("query", None)
-
-        transform_names = (
-            self.connector_manifest.config.get("transforms", "").split(",")
-            if self.connector_manifest.config.get("transforms")
-            else []
-        )
-
-        transforms = []
-        for name in transform_names:
-            transform = {"name": name}
-            transforms.append(transform)
-            for key in self.connector_manifest.config.keys():
-                if key.startswith(f"transforms.{name}."):
-                    transform[
-                        key.replace(f"transforms.{name}.", "")
-                    ] = self.connector_manifest.config[key]
-
-        return self.JdbcParser(
-            db_connection_url,
-            source_platform,
-            database_name,
-            topic_prefix,
-            query,
-            transforms,
-        )
-
-    def default_get_lineages(
-        self,
-        topic_prefix: str,
-        database_name: str,
-        source_platform: str,
-        topic_names: Optional[Iterable[str]] = None,
-        include_source_dataset: bool = True,
-    ) -> List[KafkaConnectLineage]:
-        lineages: List[KafkaConnectLineage] = []
-        if not topic_names:
-            topic_names = self.connector_manifest.topic_names
-        table_name_tuples: List[Tuple] = self.get_table_names()
-        for topic in topic_names:
-            # All good for NO_TRANSFORM or (SINGLE_TRANSFORM and KNOWN_NONTOPICROUTING_TRANSFORM) or (not SINGLE_TRANSFORM and all(KNOWN_NONTOPICROUTING_TRANSFORM))
-            source_table: str = (
-                remove_prefix(topic, topic_prefix) if topic_prefix else topic
-            )
-            # include schema name for three-level hierarchies
-            if has_three_level_hierarchy(source_platform):
-                table_name_tuple: Tuple = next(
-                    iter([t for t in table_name_tuples if t and t[-1] == source_table]),
-                    (),
-                )
-                if len(table_name_tuple) > 1:
-                    source_table = f"{table_name_tuple[-2]}.{source_table}"
-                else:
-                    include_source_dataset = False
-                    self.report_warning(
-                        self.connector_manifest.name,
-                        f"could not find schema for table {source_table}",
-                    )
-            dataset_name: str = get_dataset_name(database_name, source_table)
-            lineage = KafkaConnectLineage(
-                source_dataset=dataset_name if include_source_dataset else None,
-                source_platform=source_platform,
-                target_dataset=topic,
-                target_platform=KAFKA,
-            )
-            lineages.append(lineage)
-        return lineages
-
-    def get_table_names(self) -> List[Tuple]:
-        sep: str = "."
-        leading_quote_char: str = '"'
-        trailing_quote_char: str = leading_quote_char
-
-        table_ids: List[str] = []
-        if self.connector_manifest.tasks:
-            table_ids = (
-                ",".join(
-                    [
-                        task["config"].get("tables")
-                        for task in self.connector_manifest.tasks
-                    ]
-                )
-            ).split(",")
-            quote_method = self.connector_manifest.config.get(
-                "quote.sql.identifiers", "always"
-            )
-            if (
-                quote_method == "always"
-                and table_ids
-                and table_ids[0]
-                and table_ids[-1]
-            ):
-                leading_quote_char = table_ids[0][0]
-                trailing_quote_char = table_ids[-1][-1]
-                # This will only work for single character quotes
-        elif self.connector_manifest.config.get("table.whitelist"):
-            table_ids = self.connector_manifest.config.get("table.whitelist").split(",")  # type: ignore
-
-        # List of Tuple containing (schema, table)
-        tables: List[Tuple] = [
-            (
-                (
-                    unquote(
-                        table_id.split(sep)[-2], leading_quote_char, trailing_quote_char
-                    )
-                    if len(table_id.split(sep)) > 1
-                    else ""
-                ),
-                unquote(
-                    table_id.split(sep)[-1], leading_quote_char, trailing_quote_char
-                ),
-            )
-            for table_id in table_ids
-        ]
-        return tables
-
-    def _extract_lineages(self):
-        lineages: List[KafkaConnectLineage] = list()
-        parser = self.get_parser(self.connector_manifest)
-        source_platform = parser.source_platform
-        database_name = parser.database_name
-        query = parser.query
-        topic_prefix = parser.topic_prefix
-        transforms = parser.transforms
-        self.connector_manifest.flow_property_bag = self.connector_manifest.config
-
-        # Mask/Remove properties that may reveal credentials
-        self.connector_manifest.flow_property_bag[
-            "connection.url"
-        ] = parser.db_connection_url
-        if "connection.password" in self.connector_manifest.flow_property_bag:
-            del self.connector_manifest.flow_property_bag["connection.password"]
-        if "connection.user" in self.connector_manifest.flow_property_bag:
-            del self.connector_manifest.flow_property_bag["connection.user"]
-
-        logging.debug(
-            f"Extracting source platform: {source_platform} and database name: {database_name} from connection url "
-        )
-
-        if not self.connector_manifest.topic_names:
-            self.connector_manifest.lineages = lineages
-            return
-
-        if query:
-            # Lineage source_table can be extracted by parsing query
-            for topic in self.connector_manifest.topic_names:
-                # default method - as per earlier implementation
-                dataset_name: str = get_dataset_name(database_name, topic)
-
-                lineage = KafkaConnectLineage(
-                    source_dataset=None,
-                    source_platform=source_platform,
-                    target_dataset=topic,
-                    target_platform=KAFKA,
-                )
-                lineages.append(lineage)
-                self.report_warning(
-                    self.connector_manifest.name,
-                    "could not find input dataset, the connector has query configuration set",
-                )
-                self.connector_manifest.lineages = lineages
-                return
-
-        SINGLE_TRANSFORM = len(transforms) == 1
-        NO_TRANSFORM = len(transforms) == 0
-        UNKNOWN_TRANSFORM = any(
-            [
-                transform["type"]
-                not in self.KNOWN_TOPICROUTING_TRANSFORMS
-                + self.KNOWN_NONTOPICROUTING_TRANSFORMS
-                for transform in transforms
-            ]
-        )
-        ALL_TRANSFORMS_NON_TOPICROUTING = all(
-            [
-                transform["type"] in self.KNOWN_NONTOPICROUTING_TRANSFORMS
-                for transform in transforms
-            ]
-        )
-
-        if NO_TRANSFORM or ALL_TRANSFORMS_NON_TOPICROUTING:
-            self.connector_manifest.lineages = self.default_get_lineages(
-                database_name=database_name,
-                source_platform=source_platform,
-                topic_prefix=topic_prefix,
-            )
-            return
-
-        if SINGLE_TRANSFORM and transforms[0]["type"] == self.REGEXROUTER:
-            tables = self.get_table_names()
-            topic_names = list(self.connector_manifest.topic_names)
-
-            from java.util.regex import Pattern
-
-            for table in tables:
-                source_table: str = table[-1]
-                topic = topic_prefix + source_table if topic_prefix else source_table
-
-                transform_regex = Pattern.compile(transforms[0]["regex"])
-                transform_replacement = transforms[0]["replacement"]
-
-                matcher = transform_regex.matcher(topic)
-                if matcher.matches():
-                    topic = str(matcher.replaceFirst(transform_replacement))
-
-                # Additional check to confirm that the topic present
-                # in connector topics
-
-                if topic in self.connector_manifest.topic_names:
-                    # include schema name for three-level hierarchies
-                    if has_three_level_hierarchy(source_platform) and len(table) > 1:
-                        source_table = f"{table[-2]}.{table[-1]}"
-
-                    dataset_name = get_dataset_name(database_name, source_table)
-
-                    lineage = KafkaConnectLineage(
-                        source_dataset=dataset_name,
-                        source_platform=source_platform,
-                        target_dataset=topic,
-                        target_platform=KAFKA,
-                    )
-                    topic_names.remove(topic)
-                    lineages.append(lineage)
-
-            if topic_names:
-                lineages.extend(
-                    self.default_get_lineages(
-                        database_name=database_name,
-                        source_platform=source_platform,
-                        topic_prefix=topic_prefix,
-                        topic_names=topic_names,
-                        include_source_dataset=False,
-                    )
-                )
-                self.report_warning(
-                    self.connector_manifest.name,
-                    f"could not find input dataset, for connector topics {topic_names}",
-                )
-            self.connector_manifest.lineages = lineages
-            return
-        else:
-            include_source_dataset = True
-            if SINGLE_TRANSFORM and UNKNOWN_TRANSFORM:
-                self.report_warning(
-                    self.connector_manifest.name,
-                    f"could not find input dataset, connector has unknown transform - {transforms[0]['type']}",
-                )
-                include_source_dataset = False
-            if not SINGLE_TRANSFORM and UNKNOWN_TRANSFORM:
-                self.report_warning(
-                    self.connector_manifest.name,
-                    "could not find input dataset, connector has one or more unknown transforms",
-                )
-                include_source_dataset = False
-            lineages = self.default_get_lineages(
-                database_name=database_name,
-                source_platform=source_platform,
-                topic_prefix=topic_prefix,
-                include_source_dataset=include_source_dataset,
-            )
-            self.connector_manifest.lineages = lineages
-            return
-
-
-@dataclass
-class MongoSourceConnector:
-    # https://www.mongodb.com/docs/kafka-connector/current/source-connector/
-
-    connector_manifest: ConnectorManifest
-
-    def __init__(
-        self, connector_manifest: ConnectorManifest, config: KafkaConnectSourceConfig
-    ) -> None:
-        self.connector_manifest = connector_manifest
-        self.config = config
-        self._extract_lineages()
-
-    @dataclass
-    class MongoSourceParser:
-        db_connection_url: Optional[str]
-        source_platform: str
-        database_name: Optional[str]
-        topic_prefix: Optional[str]
-        transforms: List[str]
-
-    def get_parser(
-        self,
-        connector_manifest: ConnectorManifest,
-    ) -> MongoSourceParser:
-        parser = self.MongoSourceParser(
-            db_connection_url=connector_manifest.config.get("connection.uri"),
-            source_platform="mongodb",
-            database_name=connector_manifest.config.get("database"),
-            topic_prefix=connector_manifest.config.get("topic_prefix"),
-            transforms=(
-                connector_manifest.config["transforms"].split(",")
-                if "transforms" in connector_manifest.config
-                else []
-            ),
-        )
-
-        return parser
-
-    def _extract_lineages(self):
-        lineages: List[KafkaConnectLineage] = list()
-        parser = self.get_parser(self.connector_manifest)
-        source_platform = parser.source_platform
-        topic_naming_pattern = r"mongodb\.(\w+)\.(\w+)"
-
-        if not self.connector_manifest.topic_names:
-            return lineages
-
-        for topic in self.connector_manifest.topic_names:
-            found = re.search(re.compile(topic_naming_pattern), topic)
-
-            if found:
-                table_name = get_dataset_name(found.group(1), found.group(2))
-
-                lineage = KafkaConnectLineage(
-                    source_dataset=table_name,
-                    source_platform=source_platform,
-                    target_dataset=topic,
-                    target_platform=KAFKA,
-                )
-                lineages.append(lineage)
-        self.connector_manifest.lineages = lineages
-
-
-@dataclass
-class DebeziumSourceConnector:
-    connector_manifest: ConnectorManifest
-    report: KafkaConnectSourceReport
-
-    def __init__(
-        self,
-        connector_manifest: ConnectorManifest,
-        config: KafkaConnectSourceConfig,
-        report: KafkaConnectSourceReport,
-    ) -> None:
-        self.connector_manifest = connector_manifest
-        self.config = config
-        self.report = report
-        self._extract_lineages()
-
-    @dataclass
-    class DebeziumParser:
-        source_platform: str
-        server_name: Optional[str]
-        database_name: Optional[str]
-
-    def get_server_name(self, connector_manifest: ConnectorManifest) -> str:
-        if "topic.prefix" in connector_manifest.config:
-            return connector_manifest.config["topic.prefix"]
-        else:
-            return connector_manifest.config.get("database.server.name", "")
-
-    def get_parser(
-        self,
-        connector_manifest: ConnectorManifest,
-    ) -> DebeziumParser:
-        connector_class = connector_manifest.config.get(CONNECTOR_CLASS, "")
-
-        if connector_class == "io.debezium.connector.mysql.MySqlConnector":
-            parser = self.DebeziumParser(
-                source_platform="mysql",
-                server_name=self.get_server_name(connector_manifest),
-                database_name=None,
-            )
-        elif connector_class == "MySqlConnector":
-            parser = self.DebeziumParser(
-                source_platform="mysql",
-                server_name=self.get_server_name(connector_manifest),
-                database_name=None,
-            )
-        elif connector_class == "io.debezium.connector.mongodb.MongoDbConnector":
-            parser = self.DebeziumParser(
-                source_platform="mongodb",
-                server_name=self.get_server_name(connector_manifest),
-                database_name=None,
-            )
-        elif connector_class == "io.debezium.connector.postgresql.PostgresConnector":
-            parser = self.DebeziumParser(
-                source_platform="postgres",
-                server_name=self.get_server_name(connector_manifest),
-                database_name=connector_manifest.config.get("database.dbname"),
-            )
-        elif connector_class == "io.debezium.connector.oracle.OracleConnector":
-            parser = self.DebeziumParser(
-                source_platform="oracle",
-                server_name=self.get_server_name(connector_manifest),
-                database_name=connector_manifest.config.get("database.dbname"),
-            )
-        elif connector_class == "io.debezium.connector.sqlserver.SqlServerConnector":
-            database_name = connector_manifest.config.get(
-                "database.names"
-            ) or connector_manifest.config.get("database.dbname")
-
-            if "," in str(database_name):
-                raise Exception(
-                    f"Only one database is supported for Debezium's SQL Server connector. Found: {database_name}"
-                )
-
-            parser = self.DebeziumParser(
-                source_platform="mssql",
-                server_name=self.get_server_name(connector_manifest),
-                database_name=database_name,
-            )
-        elif connector_class == "io.debezium.connector.db2.Db2Connector":
-            parser = self.DebeziumParser(
-                source_platform="db2",
-                server_name=self.get_server_name(connector_manifest),
-                database_name=connector_manifest.config.get("database.dbname"),
-            )
-        elif connector_class == "io.debezium.connector.vitess.VitessConnector":
-            parser = self.DebeziumParser(
-                source_platform="vitess",
-                server_name=self.get_server_name(connector_manifest),
-                database_name=connector_manifest.config.get("vitess.keyspace"),
-            )
-        else:
-            raise ValueError(f"Connector class '{connector_class}' is unknown.")
-
-        return parser
-
-    def _extract_lineages(self):
-        lineages: List[KafkaConnectLineage] = list()
-
-        try:
-            parser = self.get_parser(self.connector_manifest)
-            source_platform = parser.source_platform
-            server_name = parser.server_name
-            database_name = parser.database_name
-            topic_naming_pattern = rf"({server_name})\.(\w+\.\w+)"
-
-            if not self.connector_manifest.topic_names:
-                return lineages
-
-            for topic in self.connector_manifest.topic_names:
-                found = re.search(re.compile(topic_naming_pattern), topic)
-
-                if found:
-                    table_name = get_dataset_name(database_name, found.group(2))
-
-                    lineage = KafkaConnectLineage(
-                        source_dataset=table_name,
-                        source_platform=source_platform,
-                        target_dataset=topic,
-                        target_platform=KAFKA,
-                    )
-                    lineages.append(lineage)
-            self.connector_manifest.lineages = lineages
-        except Exception as e:
-            self.report.report_warning(
-                self.connector_manifest.name, f"Error resolving lineage: {e}"
-            )
-
-        return
-
-
-@dataclass
-class BigQuerySinkConnector:
-    connector_manifest: ConnectorManifest
-    report: KafkaConnectSourceReport
-
-    def __init__(
-        self, connector_manifest: ConnectorManifest, report: KafkaConnectSourceReport
-    ) -> None:
-        self.connector_manifest = connector_manifest
-        self.report = report
-        self._extract_lineages()
-
-    @dataclass
-    class BQParser:
-        project: str
-        target_platform: str
-        sanitizeTopics: str
-        transforms: list
-        topicsToTables: Optional[str] = None
-        datasets: Optional[str] = None
-        defaultDataset: Optional[str] = None
-        version: str = "v1"
-
-    def report_warning(self, key: str, reason: str) -> None:
-        logger.warning(f"{key}: {reason}")
-        self.report.report_warning(key, reason)
-
-    def get_parser(
-        self,
-        connector_manifest: ConnectorManifest,
-    ) -> BQParser:
-        project = connector_manifest.config["project"]
-        sanitizeTopics = connector_manifest.config.get("sanitizeTopics", "false")
-        transform_names = (
-            self.connector_manifest.config.get("transforms", "").split(",")
-            if self.connector_manifest.config.get("transforms")
-            else []
-        )
-        transforms = []
-        for name in transform_names:
-            transform = {"name": name}
-            transforms.append(transform)
-            for key in self.connector_manifest.config.keys():
-                if key.startswith(f"transforms.{name}."):
-                    transform[
-                        key.replace(f"transforms.{name}.", "")
-                    ] = self.connector_manifest.config[key]
-
-        if "defaultDataset" in connector_manifest.config:
-            defaultDataset = connector_manifest.config["defaultDataset"]
-            return self.BQParser(
-                project=project,
-                defaultDataset=defaultDataset,
-                target_platform="bigquery",
-                sanitizeTopics=sanitizeTopics.lower() == "true",
-                version="v2",
-                transforms=transforms,
-            )
-        else:
-            # version 1.6.x and similar configs supported
-            datasets = connector_manifest.config["datasets"]
-            topicsToTables = connector_manifest.config.get("topicsToTables")
-
-            return self.BQParser(
-                project=project,
-                topicsToTables=topicsToTables,
-                datasets=datasets,
-                target_platform="bigquery",
-                sanitizeTopics=sanitizeTopics.lower() == "true",
-                transforms=transforms,
-            )
-
-    def get_list(self, property: str) -> Iterable[Tuple[str, str]]:
-        entries = property.split(",")
-        for entry in entries:
-            key, val = entry.rsplit("=")
-            yield (key.strip(), val.strip())
-
-    def get_dataset_for_topic_v1(self, topic: str, parser: BQParser) -> Optional[str]:
-        topicregex_dataset_map: Dict[str, str] = dict(self.get_list(parser.datasets))  # type: ignore
-        from java.util.regex import Pattern
-
-        for pattern, dataset in topicregex_dataset_map.items():
-            patternMatcher = Pattern.compile(pattern).matcher(topic)
-            if patternMatcher.matches():
-                return dataset
-        return None
-
-    def sanitize_table_name(self, table_name):
-        table_name = re.sub("[^a-zA-Z0-9_]", "_", table_name)
-        if re.match("^[^a-zA-Z_].*", table_name):
-            table_name = "_" + table_name
-
-        return table_name
-
-    def get_dataset_table_for_topic(
-        self, topic: str, parser: BQParser
-    ) -> Optional[str]:
-        if parser.version == "v2":
-            dataset = parser.defaultDataset
-            parts = topic.split(":")
-            if len(parts) == 2:
-                dataset = parts[0]
-                table = parts[1]
-            else:
-                table = parts[0]
-        else:
-            dataset = self.get_dataset_for_topic_v1(topic, parser)
-            if dataset is None:
-                return None
-
-            table = topic
-            if parser.topicsToTables:
-                topicregex_table_map: Dict[str, str] = dict(
-                    self.get_list(parser.topicsToTables)  # type: ignore
-                )
-                from java.util.regex import Pattern
-
-                for pattern, tbl in topicregex_table_map.items():
-                    patternMatcher = Pattern.compile(pattern).matcher(topic)
-                    if patternMatcher.matches():
-                        table = tbl
-                        break
-
-        if parser.sanitizeTopics:
-            table = self.sanitize_table_name(table)
-        return f"{dataset}.{table}"
-
-    def apply_transformations(
-        self, topic: str, transforms: List[Dict[str, str]]
-    ) -> str:
-        for transform in transforms:
-            if transform["type"] == "org.apache.kafka.connect.transforms.RegexRouter":
-                regex = transform["regex"]
-                replacement = transform["replacement"]
-                pattern = re.compile(regex)
-                if pattern.match(topic):
-                    topic = pattern.sub(replacement, topic, count=1)
-        return topic
-
-    def _extract_lineages(self):
-        lineages: List[KafkaConnectLineage] = list()
-        parser = self.get_parser(self.connector_manifest)
-        if not parser:
-            return lineages
-        target_platform = parser.target_platform
-        project = parser.project
-        transforms = parser.transforms
-        self.connector_manifest.flow_property_bag = self.connector_manifest.config
-        # Mask/Remove properties that may reveal credentials
-        if "keyfile" in self.connector_manifest.flow_property_bag:
-            del self.connector_manifest.flow_property_bag["keyfile"]
-
-        for topic in self.connector_manifest.topic_names:
-            transformed_topic = self.apply_transformations(topic, transforms)
-            dataset_table = self.get_dataset_table_for_topic(transformed_topic, parser)
-            if dataset_table is None:
-                self.report_warning(
-                    self.connector_manifest.name,
-                    f"could not find target dataset for topic {transformed_topic}, please check your connector configuration",
-                )
-                continue
-            target_dataset = f"{project}.{dataset_table}"
-
-            lineages.append(
-                KafkaConnectLineage(
-                    source_dataset=transformed_topic,
-                    source_platform=KAFKA,
-                    target_dataset=target_dataset,
-                    target_platform=target_platform,
-                )
-            )
-        self.connector_manifest.lineages = lineages
-        return
-
-
-@dataclass
-class SnowflakeSinkConnector:
-    connector_manifest: ConnectorManifest
-    report: KafkaConnectSourceReport
-
-    def __init__(
-        self, connector_manifest: ConnectorManifest, report: KafkaConnectSourceReport
-    ) -> None:
-        self.connector_manifest = connector_manifest
-        self.report = report
-        self._extract_lineages()
-
-    @dataclass
-    class SnowflakeParser:
-        database_name: str
-        schema_name: str
-        topics_to_tables: Dict[str, str]
-
-    def report_warning(self, key: str, reason: str) -> None:
-        logger.warning(f"{key}: {reason}")
-        self.report.report_warning(key, reason)
-
-    def get_table_name_from_topic_name(self, topic_name: str) -> str:
-        """
-        This function converts the topic name to a valid Snowflake table name using some rules.
-        Refer below link for more info
-        https://docs.snowflake.com/en/user-guide/kafka-connector-overview#target-tables-for-kafka-topics
-        """
-        table_name = re.sub("[^a-zA-Z0-9_]", "_", topic_name)
-        if re.match("^[^a-zA-Z_].*", table_name):
-            table_name = "_" + table_name
-        # Connector  may append original topic's hash code as suffix for conflict resolution
-        # if generated table names for 2 topics are similar. This corner case is not handled here.
-        # Note that Snowflake recommends to choose topic names that follow the rules for
-        # Snowflake identifier names so this case is not recommended by snowflake.
-        return table_name
-
-    def get_parser(
-        self,
-        connector_manifest: ConnectorManifest,
-    ) -> SnowflakeParser:
-        database_name = connector_manifest.config["snowflake.database.name"]
-        schema_name = connector_manifest.config["snowflake.schema.name"]
-
-        # Fetch user provided topic to table map
-        provided_topics_to_tables: Dict[str, str] = {}
-        if connector_manifest.config.get("snowflake.topic2table.map"):
-            for each in connector_manifest.config["snowflake.topic2table.map"].split(
-                ","
-            ):
-                topic, table = each.split(":")
-                provided_topics_to_tables[topic.strip()] = table.strip()
-
-        topics_to_tables: Dict[str, str] = {}
-        # Extract lineage for only those topics whose data ingestion started
-        for topic in connector_manifest.topic_names:
-            if topic in provided_topics_to_tables:
-                # If user provided which table to get mapped with this topic
-                topics_to_tables[topic] = provided_topics_to_tables[topic]
-            else:
-                # Else connector converts topic name to a valid Snowflake table name.
-                topics_to_tables[topic] = self.get_table_name_from_topic_name(topic)
-
-        return self.SnowflakeParser(
-            database_name=database_name,
-            schema_name=schema_name,
-            topics_to_tables=topics_to_tables,
-        )
-
-    def _extract_lineages(self):
-        self.connector_manifest.flow_property_bag = self.connector_manifest.config
-
-        # For all snowflake sink connector properties, refer below link
-        # https://docs.snowflake.com/en/user-guide/kafka-connector-install#configuring-the-kafka-connector
-        # remove private keys, secrets from properties
-        secret_properties = [
-            "snowflake.private.key",
-            "snowflake.private.key.passphrase",
-            "value.converter.basic.auth.user.info",
-        ]
-        for k in secret_properties:
-            if k in self.connector_manifest.flow_property_bag:
-                del self.connector_manifest.flow_property_bag[k]
-
-        lineages: List[KafkaConnectLineage] = list()
-        parser = self.get_parser(self.connector_manifest)
-
-        for topic, table in parser.topics_to_tables.items():
-            target_dataset = f"{parser.database_name}.{parser.schema_name}.{table}"
-            lineages.append(
-                KafkaConnectLineage(
-                    source_dataset=topic,
-                    source_platform=KAFKA,
-                    target_dataset=target_dataset,
-                    target_platform="snowflake",
-                )
-            )
-
-        self.connector_manifest.lineages = lineages
-        return
-
-
-@dataclass
-class ConfluentS3SinkConnector:
-    connector_manifest: ConnectorManifest
-
-    def __init__(
-        self, connector_manifest: ConnectorManifest, report: KafkaConnectSourceReport
-    ) -> None:
-        self.connector_manifest = connector_manifest
-        self.report = report
-        self._extract_lineages()
-
-    @dataclass
-    class S3SinkParser:
-        target_platform: str
-        bucket: str
-        topics_dir: str
-        topics: Iterable[str]
-
-    def _get_parser(self, connector_manifest: ConnectorManifest) -> S3SinkParser:
-        # https://docs.confluent.io/kafka-connectors/s3-sink/current/configuration_options.html#s3
-        bucket = connector_manifest.config.get("s3.bucket.name")
-        if not bucket:
-            raise ValueError(
-                "Could not find 's3.bucket.name' in connector configuration"
-            )
-
-        # https://docs.confluent.io/kafka-connectors/s3-sink/current/configuration_options.html#storage
-        topics_dir = connector_manifest.config.get("topics.dir", "topics")
-
-        return self.S3SinkParser(
-            target_platform="s3",
-            bucket=bucket,
-            topics_dir=topics_dir,
-            topics=connector_manifest.topic_names,
-        )
-
-    def _extract_lineages(self):
-        self.connector_manifest.flow_property_bag = self.connector_manifest.config
-
-        # remove keys, secrets from properties
-        secret_properties = [
-            "aws.access.key.id",
-            "aws.secret.access.key",
-            "s3.sse.customer.key",
-            "s3.proxy.password",
-        ]
-        for k in secret_properties:
-            if k in self.connector_manifest.flow_property_bag:
-                del self.connector_manifest.flow_property_bag[k]
-
-        try:
-            parser = self._get_parser(self.connector_manifest)
-
-            lineages: List[KafkaConnectLineage] = list()
-            for topic in parser.topics:
-                target_dataset = f"{parser.bucket}/{parser.topics_dir}/{topic}"
-
-                lineages.append(
-                    KafkaConnectLineage(
-                        source_dataset=topic,
-                        source_platform="kafka",
-                        target_dataset=target_dataset,
-                        target_platform=parser.target_platform,
-                    )
-                )
-            self.connector_manifest.lineages = lineages
-        except Exception as e:
-            self.report.report_warning(
-                self.connector_manifest.name, f"Error resolving lineage: {e}"
-            )
-
-        return
-
-
-def transform_connector_config(
-    connector_config: Dict, provided_configs: List[ProvidedConfig]
-) -> None:
-    """This method will update provided configs in connector config values, if any"""
-    lookupsByProvider = {}
-    for pconfig in provided_configs:
-        lookupsByProvider[f"${{{pconfig.provider}:{pconfig.path_key}}}"] = pconfig.value
-    for k, v in connector_config.items():
-        for key, value in lookupsByProvider.items():
-            if key in v:
-                connector_config[k] = connector_config[k].replace(key, value)
-
-
-@platform_name("Kafka Connect")
-@config_class(KafkaConnectSourceConfig)
-@support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
-@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
-@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
-class KafkaConnectSource(StatefulIngestionSourceBase):
-    config: KafkaConnectSourceConfig
-    report: KafkaConnectSourceReport
-    platform: str = "kafka-connect"
-
-    def __init__(self, config: KafkaConnectSourceConfig, ctx: PipelineContext):
-        super().__init__(config, ctx)
-        self.config = config
-        self.report = KafkaConnectSourceReport()
-        self.session = requests.Session()
-        self.session.headers.update(
-            {
-                "Accept": "application/json",
-                "Content-Type": "application/json",
-            }
-        )
-
-        # Test the connection
-        if self.config.username is not None and self.config.password is not None:
-            logger.info(
-                f"Connecting to {self.config.connect_uri} with Authentication..."
-            )
-            self.session.auth = (self.config.username, self.config.password)
-
-        test_response = self.session.get(f"{self.config.connect_uri}")
-        test_response.raise_for_status()
-        logger.info(f"Connection to {self.config.connect_uri} is ok")
-        if not jpype.isJVMStarted():
-            jpype.startJVM()
-
-    @classmethod
-    def create(cls, config_dict: dict, ctx: PipelineContext) -> Source:
-        config = KafkaConnectSourceConfig.parse_obj(config_dict)
-        return cls(config, ctx)
-
-    def get_connectors_manifest(self) -> List[ConnectorManifest]:
-        """Get Kafka Connect connectors manifest using REST API.
-        Enrich with lineages metadata.
-        """
-        connectors_manifest = list()
-
-        connector_response = self.session.get(
-            f"{self.config.connect_uri}/connectors",
-        )
-
-        payload = connector_response.json()
-
-        for c in payload:
-            connector_url = f"{self.config.connect_uri}/connectors/{c}"
-            connector_response = self.session.get(connector_url)
-            manifest = connector_response.json()
-            connector_manifest = ConnectorManifest(**manifest)
-            if not self.config.connector_patterns.allowed(connector_manifest.name):
-                self.report.report_dropped(connector_manifest.name)
-                continue
-
-            if self.config.provided_configs:
-                transform_connector_config(
-                    connector_manifest.config, self.config.provided_configs
-                )
-            # Initialize connector lineages
-            connector_manifest.lineages = list()
-            connector_manifest.url = connector_url
-
-            topics = self.session.get(
-                f"{self.config.connect_uri}/connectors/{c}/topics",
-            ).json()
-
-            connector_manifest.topic_names = topics[c]["topics"]
-
-            # Populate Source Connector metadata
-            if connector_manifest.type == SOURCE:
-                tasks = self.session.get(
-                    f"{self.config.connect_uri}/connectors/{c}/tasks",
-                ).json()
-
-                connector_manifest.tasks = tasks
-
-                # JDBC source connector lineages
-                if connector_manifest.config.get(CONNECTOR_CLASS).__eq__(
-                    "io.confluent.connect.jdbc.JdbcSourceConnector"
-                ):
-                    connector_manifest = ConfluentJDBCSourceConnector(
-                        connector_manifest=connector_manifest,
-                        config=self.config,
-                        report=self.report,
-                    ).connector_manifest
-                elif connector_manifest.config.get(CONNECTOR_CLASS, "").startswith(
-                    "io.debezium.connector"
-                ):
-                    connector_manifest = DebeziumSourceConnector(
-                        connector_manifest=connector_manifest,
-                        config=self.config,
-                        report=self.report,
-                    ).connector_manifest
-                elif (
-                    connector_manifest.config.get(CONNECTOR_CLASS, "")
-                    == "com.mongodb.kafka.connect.MongoSourceConnector"
-                ):
-                    connector_manifest = MongoSourceConnector(
-                        connector_manifest=connector_manifest, config=self.config
-                    ).connector_manifest
-                else:
-                    # Find the target connector object in the list, or log an error if unknown.
-                    target_connector = None
-                    for connector in self.config.generic_connectors:
-                        if connector.connector_name == connector_manifest.name:
-                            target_connector = connector
-                            break
-                    if not target_connector:
-                        logger.warning(
-                            f"Detected undefined connector {connector_manifest.name}, which is not in the customized connector list. Please refer to Kafka Connect ingestion recipe to define this customized connector."
-                        )
-                        continue
-
-                    for topic in topics:
-                        lineage = KafkaConnectLineage(
-                            source_dataset=target_connector.source_dataset,
-                            source_platform=target_connector.source_platform,
-                            target_dataset=topic,
-                            target_platform=KAFKA,
-                        )
-
-                    connector_manifest.lineages.append(lineage)
-
-            if connector_manifest.type == SINK:
-                if connector_manifest.config.get(CONNECTOR_CLASS).__eq__(
-                    "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector"
-                ):
-                    connector_manifest = BigQuerySinkConnector(
-                        connector_manifest=connector_manifest, report=self.report
-                    ).connector_manifest
-                elif connector_manifest.config.get("connector.class").__eq__(
-                    "io.confluent.connect.s3.S3SinkConnector"
-                ):
-                    connector_manifest = ConfluentS3SinkConnector(
-                        connector_manifest=connector_manifest, report=self.report
-                    ).connector_manifest
-                elif connector_manifest.config.get("connector.class").__eq__(
-                    "com.snowflake.kafka.connector.SnowflakeSinkConnector"
-                ):
-                    connector_manifest = SnowflakeSinkConnector(
-                        connector_manifest=connector_manifest, report=self.report
-                    ).connector_manifest
-                else:
-                    self.report.report_dropped(connector_manifest.name)
-                    logger.warning(
-                        f"Skipping connector {connector_manifest.name}. Lineage for  Connector not yet implemented"
-                    )
-                pass
-
-            connectors_manifest.append(connector_manifest)
-
-        return connectors_manifest
-
-    def construct_flow_workunit(self, connector: ConnectorManifest) -> MetadataWorkUnit:
-        connector_name = connector.name
-        connector_type = connector.type
-        connector_class = connector.config.get(CONNECTOR_CLASS)
-        flow_property_bag = connector.flow_property_bag
-        # connector_url = connector.url  # NOTE: this will expose connector credential when used
-        flow_urn = builder.make_data_flow_urn(
-            self.platform,
-            connector_name,
-            self.config.env,
-            self.config.platform_instance,
-        )
-
-        return MetadataChangeProposalWrapper(
-            entityUrn=flow_urn,
-            aspect=models.DataFlowInfoClass(
-                name=connector_name,
-                description=f"{connector_type.capitalize()} connector using `{connector_class}` plugin.",
-                customProperties=flow_property_bag,
-                # externalUrl=connector_url, # NOTE: this will expose connector credential when used
-            ),
-        ).as_workunit()
-
-    def construct_job_workunits(
-        self, connector: ConnectorManifest
-    ) -> Iterable[MetadataWorkUnit]:
-        connector_name = connector.name
-        flow_urn = builder.make_data_flow_urn(
-            self.platform,
-            connector_name,
-            self.config.env,
-            self.config.platform_instance,
-        )
-
-        lineages = connector.lineages
-        if lineages:
-            for lineage in lineages:
-                source_dataset = lineage.source_dataset
-                source_platform = lineage.source_platform
-                target_dataset = lineage.target_dataset
-                target_platform = lineage.target_platform
-                job_property_bag = lineage.job_property_bag
-
-                source_platform_instance = get_platform_instance(
-                    self.config, connector_name, source_platform
-                )
-                target_platform_instance = get_platform_instance(
-                    self.config, connector_name, target_platform
-                )
-
-                job_id = self.get_job_id(lineage, connector, self.config)
-                job_urn = builder.make_data_job_urn_with_flow(flow_urn, job_id)
-
-                inlets = (
-                    [
-                        self.make_lineage_dataset_urn(
-                            source_platform, source_dataset, source_platform_instance
-                        )
-                    ]
-                    if source_dataset
-                    else []
-                )
-                outlets = [
-                    self.make_lineage_dataset_urn(
-                        target_platform, target_dataset, target_platform_instance
-                    )
-                ]
-
-                yield MetadataChangeProposalWrapper(
-                    entityUrn=job_urn,
-                    aspect=models.DataJobInfoClass(
-                        name=f"{connector_name}:{job_id}",
-                        type="COMMAND",
-                        customProperties=job_property_bag,
-                    ),
-                ).as_workunit()
-
-                yield MetadataChangeProposalWrapper(
-                    entityUrn=job_urn,
-                    aspect=models.DataJobInputOutputClass(
-                        inputDatasets=inlets,
-                        outputDatasets=outlets,
-                    ),
-                ).as_workunit()
-
-    def get_job_id(
-        self,
-        lineage: KafkaConnectLineage,
-        connector: ConnectorManifest,
-        config: KafkaConnectSourceConfig,
-    ) -> str:
-        connector_class = connector.config.get(CONNECTOR_CLASS)
-
-        # Note - This block is only to maintain backward compatibility of Job URN
-        if (
-            connector_class
-            and connector.type == SOURCE
-            and (
-                "JdbcSourceConnector" in connector_class
-                or connector_class.startswith("io.debezium.connector")
-            )
-            and lineage.source_dataset
-            and config.connect_to_platform_map
-            and config.connect_to_platform_map.get(connector.name)
-            and config.connect_to_platform_map[connector.name].get(
-                lineage.source_platform
-            )
-        ):
-            return f"{config.connect_to_platform_map[connector.name][lineage.source_platform]}.{lineage.source_dataset}"
-
-        return (
-            lineage.source_dataset
-            if lineage.source_dataset
-            else f"unknown_source.{lineage.target_dataset}"
-        )
-
-    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
-        return [
-            *super().get_workunit_processors(),
-            StaleEntityRemovalHandler.create(
-                self, self.config, self.ctx
-            ).workunit_processor,
-        ]
-
-    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
-        connectors_manifest = self.get_connectors_manifest()
-        for connector in connectors_manifest:
-            name = connector.name
-
-            yield self.construct_flow_workunit(connector)
-            yield from self.construct_job_workunits(connector)
-            self.report.report_connector_scanned(name)
-
-    def get_report(self) -> KafkaConnectSourceReport:
-        return self.report
-
-    def make_lineage_dataset_urn(
-        self, platform: str, name: str, platform_instance: Optional[str]
-    ) -> str:
-        if self.config.convert_lineage_urns_to_lowercase:
-            name = name.lower()
-
-        return builder.make_dataset_urn_with_platform_instance(
-            platform, name, platform_instance, self.config.env
-        )
-
-
-# TODO: Find a more automated way to discover new platforms with 3 level naming hierarchy.
-def has_three_level_hierarchy(platform: str) -> bool:
-    return platform in ["postgres", "trino", "redshift", "snowflake"]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/common.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/common.py
new file mode 100644
index 0000000000000..36f6a96c0d408
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/common.py
@@ -0,0 +1,202 @@
+import logging
+from dataclasses import dataclass, field
+from typing import Dict, Iterable, List, Optional
+
+from pydantic.fields import Field
+
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.source_common import (
+    DatasetLineageProviderConfigBase,
+    PlatformInstanceConfigMixin,
+)
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalSourceReport,
+    StatefulStaleMetadataRemovalConfig,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionConfigBase,
+)
+
+logger = logging.getLogger(__name__)
+
+KAFKA = "kafka"
+SOURCE = "source"
+SINK = "sink"
+CONNECTOR_CLASS = "connector.class"
+
+
+class ProvidedConfig(ConfigModel):
+    provider: str
+    path_key: str
+    value: str
+
+
+class GenericConnectorConfig(ConfigModel):
+    connector_name: str
+    source_dataset: str
+    source_platform: str
+
+
+class KafkaConnectSourceConfig(
+    PlatformInstanceConfigMixin,
+    DatasetLineageProviderConfigBase,
+    StatefulIngestionConfigBase,
+):
+    # See the Connect REST Interface for details
+    # https://docs.confluent.io/platform/current/connect/references/restapi.html#
+    connect_uri: str = Field(
+        default="http://localhost:8083/", description="URI to connect to."
+    )
+    username: Optional[str] = Field(default=None, description="Kafka Connect username.")
+    password: Optional[str] = Field(default=None, description="Kafka Connect password.")
+    cluster_name: Optional[str] = Field(
+        default="connect-cluster", description="Cluster to ingest from."
+    )
+    # convert lineage dataset's urns to lowercase
+    convert_lineage_urns_to_lowercase: bool = Field(
+        default=False,
+        description="Whether to convert the urns of ingested lineage dataset to lowercase",
+    )
+    connector_patterns: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="regex patterns for connectors to filter for ingestion.",
+    )
+    provided_configs: Optional[List[ProvidedConfig]] = Field(
+        default=None, description="Provided Configurations"
+    )
+    connect_to_platform_map: Optional[Dict[str, Dict[str, str]]] = Field(
+        default=None,
+        description='Platform instance mapping when multiple instances for a platform is available. Entry for a platform should be in either `platform_instance_map` or `connect_to_platform_map`. e.g.`connect_to_platform_map: { "postgres-connector-finance-db": "postgres": "core_finance_instance" }`',
+    )
+    platform_instance_map: Optional[Dict[str, str]] = Field(
+        default=None,
+        description='Platform instance mapping to use when constructing URNs. e.g.`platform_instance_map: { "hive": "warehouse" }`',
+    )
+    generic_connectors: List[GenericConnectorConfig] = Field(
+        default=[],
+        description="Provide lineage graph for sources connectors other than Confluent JDBC Source Connector, Debezium Source Connector, and Mongo Source Connector",
+    )
+
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
+
+
+@dataclass
+class KafkaConnectSourceReport(StaleEntityRemovalSourceReport):
+    connectors_scanned: int = 0
+    filtered: List[str] = field(default_factory=list)
+
+    def report_connector_scanned(self, connector: str) -> None:
+        self.connectors_scanned += 1
+
+    def report_dropped(self, connector: str) -> None:
+        self.filtered.append(connector)
+
+
+@dataclass
+class KafkaConnectLineage:
+    """Class to store Kafka Connect lineage mapping, Each instance is potential DataJob"""
+
+    source_platform: str
+    target_dataset: str
+    target_platform: str
+    job_property_bag: Optional[Dict[str, str]] = None
+    source_dataset: Optional[str] = None
+
+
+@dataclass
+class ConnectorManifest:
+    """Each instance is potential DataFlow"""
+
+    name: str
+    type: str
+    config: Dict
+    tasks: Dict
+    url: Optional[str] = None
+    flow_property_bag: Optional[Dict[str, str]] = None
+    lineages: List[KafkaConnectLineage] = field(default_factory=list)
+    topic_names: Iterable[str] = field(default_factory=list)
+
+
+def remove_prefix(text: str, prefix: str) -> str:
+    if text.startswith(prefix):
+        index = len(prefix)
+        return text[index:]
+    return text
+
+
+def unquote(
+    string: str, leading_quote: str = '"', trailing_quote: Optional[str] = None
+) -> str:
+    """
+    If string starts and ends with a quote, unquote it
+    """
+    trailing_quote = trailing_quote if trailing_quote else leading_quote
+    if string.startswith(leading_quote) and string.endswith(trailing_quote):
+        string = string[1:-1]
+    return string
+
+
+def get_dataset_name(
+    database_name: Optional[str],
+    source_table: str,
+) -> str:
+    if database_name:
+        dataset_name = database_name + "." + source_table
+    else:
+        dataset_name = source_table
+
+    return dataset_name
+
+
+def get_platform_instance(
+    config: KafkaConnectSourceConfig, connector_name: str, platform: str
+) -> Optional[str]:
+    instance_name = None
+    if (
+        config.connect_to_platform_map
+        and config.connect_to_platform_map.get(connector_name)
+        and config.connect_to_platform_map[connector_name].get(platform)
+    ):
+        instance_name = config.connect_to_platform_map[connector_name][platform]
+        if config.platform_instance_map and config.platform_instance_map.get(platform):
+            logger.warning(
+                f"Same source platform {platform} configured in both platform_instance_map and connect_to_platform_map."
+                "Will prefer connector specific platform instance from connect_to_platform_map."
+            )
+    elif config.platform_instance_map and config.platform_instance_map.get(platform):
+        instance_name = config.platform_instance_map[platform]
+    logger.info(
+        f"Instance name assigned is: {instance_name} for Connector Name {connector_name} and platform {platform}"
+    )
+    return instance_name
+
+
+def transform_connector_config(
+    connector_config: Dict, provided_configs: List[ProvidedConfig]
+) -> None:
+    """This method will update provided configs in connector config values, if any"""
+    lookupsByProvider = {}
+    for pconfig in provided_configs:
+        lookupsByProvider[f"${{{pconfig.provider}:{pconfig.path_key}}}"] = pconfig.value
+    for k, v in connector_config.items():
+        for key, value in lookupsByProvider.items():
+            if key in v:
+                connector_config[k] = connector_config[k].replace(key, value)
+
+
+# TODO: Find a more automated way to discover new platforms with 3 level naming hierarchy.
+def has_three_level_hierarchy(platform: str) -> bool:
+    return platform in ["postgres", "trino", "redshift", "snowflake"]
+
+
+@dataclass
+class BaseConnector:
+    connector_manifest: ConnectorManifest
+    config: KafkaConnectSourceConfig
+    report: KafkaConnectSourceReport
+
+    def extract_lineages(self) -> List[KafkaConnectLineage]:
+        return []
+
+    def extract_flow_property_bag(self) -> Optional[Dict[str, str]]:
+        return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/kafka_connect.py
new file mode 100644
index 0000000000000..fa6b614c4b52a
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/kafka_connect.py
@@ -0,0 +1,367 @@
+import logging
+from typing import Iterable, List, Optional, Type
+
+import jpype
+import jpype.imports
+import requests
+
+import datahub.emitter.mce_builder as builder
+import datahub.metadata.schema_classes as models
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SourceCapability,
+    SupportStatus,
+    capability,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.kafka_connect.common import (
+    CONNECTOR_CLASS,
+    SINK,
+    SOURCE,
+    BaseConnector,
+    ConnectorManifest,
+    KafkaConnectLineage,
+    KafkaConnectSourceConfig,
+    KafkaConnectSourceReport,
+    get_platform_instance,
+    transform_connector_config,
+)
+from datahub.ingestion.source.kafka_connect.sink_connectors import (
+    BIGQUERY_SINK_CONNECTOR_CLASS,
+    S3_SINK_CONNECTOR_CLASS,
+    SNOWFLAKE_SINK_CONNECTOR_CLASS,
+    BigQuerySinkConnector,
+    ConfluentS3SinkConnector,
+    SnowflakeSinkConnector,
+)
+from datahub.ingestion.source.kafka_connect.source_connectors import (
+    DEBEZIUM_SOURCE_CONNECTOR_PREFIX,
+    JDBC_SOURCE_CONNECTOR_CLASS,
+    MONGO_SOURCE_CONNECTOR_CLASS,
+    ConfigDrivenSourceConnector,
+    ConfluentJDBCSourceConnector,
+    DebeziumSourceConnector,
+    MongoSourceConnector,
+)
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionSourceBase,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@platform_name("Kafka Connect")
+@config_class(KafkaConnectSourceConfig)
+@support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
+@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
+class KafkaConnectSource(StatefulIngestionSourceBase):
+    config: KafkaConnectSourceConfig
+    report: KafkaConnectSourceReport
+    platform: str = "kafka-connect"
+
+    def __init__(self, config: KafkaConnectSourceConfig, ctx: PipelineContext):
+        super().__init__(config, ctx)
+        self.config = config
+        self.report = KafkaConnectSourceReport()
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                "Accept": "application/json",
+                "Content-Type": "application/json",
+            }
+        )
+
+        # Test the connection
+        if self.config.username is not None and self.config.password is not None:
+            logger.info(
+                f"Connecting to {self.config.connect_uri} with Authentication..."
+            )
+            self.session.auth = (self.config.username, self.config.password)
+
+        test_response = self.session.get(f"{self.config.connect_uri}/connectors")
+        test_response.raise_for_status()
+        logger.info(f"Connection to {self.config.connect_uri} is ok")
+        if not jpype.isJVMStarted():
+            jpype.startJVM()
+
+    @classmethod
+    def create(cls, config_dict: dict, ctx: PipelineContext) -> Source:
+        config = KafkaConnectSourceConfig.parse_obj(config_dict)
+        return cls(config, ctx)
+
+    def get_connectors_manifest(self) -> Iterable[ConnectorManifest]:
+        """Get Kafka Connect connectors manifest using REST API.
+        Enrich with lineages metadata.
+        """
+
+        connector_response = self.session.get(
+            f"{self.config.connect_uri}/connectors",
+        )
+
+        payload = connector_response.json()
+
+        for connector_name in payload:
+            connector_url = f"{self.config.connect_uri}/connectors/{connector_name}"
+            connector_manifest = self._get_connector_manifest(
+                connector_name, connector_url
+            )
+            if (
+                connector_manifest is None
+                or not self.config.connector_patterns.allowed(connector_manifest.name)
+            ):
+                self.report.report_dropped(connector_name)
+                continue
+
+            if self.config.provided_configs:
+                transform_connector_config(
+                    connector_manifest.config, self.config.provided_configs
+                )
+            connector_manifest.url = connector_url
+            connector_manifest.topic_names = self._get_connector_topics(connector_name)
+            connector_class_value = connector_manifest.config.get(CONNECTOR_CLASS) or ""
+
+            class_type: Type[BaseConnector] = BaseConnector
+
+            # Populate Source Connector metadata
+            if connector_manifest.type == SOURCE:
+                connector_manifest.tasks = self._get_connector_tasks(connector_name)
+
+                # JDBC source connector lineages
+                if connector_class_value == JDBC_SOURCE_CONNECTOR_CLASS:
+                    class_type = ConfluentJDBCSourceConnector
+                elif connector_class_value.startswith(DEBEZIUM_SOURCE_CONNECTOR_PREFIX):
+                    class_type = DebeziumSourceConnector
+                elif connector_class_value == MONGO_SOURCE_CONNECTOR_CLASS:
+                    class_type = MongoSourceConnector
+                elif any(
+                    [
+                        connector.connector_name == connector_manifest.name
+                        for connector in self.config.generic_connectors
+                    ]
+                ):
+                    class_type = ConfigDrivenSourceConnector
+                else:
+                    self.report.report_dropped(connector_manifest.name)
+                    self.report.warning(
+                        "Lineage for Source Connector not supported. "
+                        "Please refer to Kafka Connect docs to use `generic_connectors` config.",
+                        context=f"{connector_manifest.name} of type {connector_class_value}",
+                    )
+                    continue
+            elif connector_manifest.type == SINK:
+                if connector_class_value == BIGQUERY_SINK_CONNECTOR_CLASS:
+                    class_type = BigQuerySinkConnector
+                elif connector_class_value == S3_SINK_CONNECTOR_CLASS:
+                    class_type = ConfluentS3SinkConnector
+                elif connector_class_value == SNOWFLAKE_SINK_CONNECTOR_CLASS:
+                    class_type = SnowflakeSinkConnector
+                else:
+                    self.report.report_dropped(connector_manifest.name)
+                    self.report.warning(
+                        "Lineage for Sink Connector not supported.",
+                        context=f"{connector_manifest.name} of type {connector_class_value}",
+                    )
+
+            connector_class = class_type(connector_manifest, self.config, self.report)
+            connector_manifest.lineages = connector_class.extract_lineages()
+            connector_manifest.flow_property_bag = (
+                connector_class.extract_flow_property_bag()
+            )
+
+            yield connector_manifest
+
+    def _get_connector_manifest(
+        self, connector_name: str, connector_url: str
+    ) -> Optional[ConnectorManifest]:
+        try:
+            connector_response = self.session.get(connector_url)
+            connector_response.raise_for_status()
+        except Exception as e:
+            self.report.warning(
+                "Failed to get connector details", connector_name, exc=e
+            )
+            return None
+        manifest = connector_response.json()
+        connector_manifest = ConnectorManifest(**manifest)
+        return connector_manifest
+
+    def _get_connector_tasks(self, connector_name: str) -> dict:
+        try:
+            response = self.session.get(
+                f"{self.config.connect_uri}/connectors/{connector_name}/tasks",
+            )
+            response.raise_for_status()
+        except Exception as e:
+            self.report.warning(
+                "Error getting connector tasks", context=connector_name, exc=e
+            )
+            return {}
+
+        return response.json()
+
+    def _get_connector_topics(self, connector_name: str) -> List[str]:
+        try:
+            response = self.session.get(
+                f"{self.config.connect_uri}/connectors/{connector_name}/topics",
+            )
+            response.raise_for_status()
+        except Exception as e:
+            self.report.warning(
+                "Error getting connector topics", context=connector_name, exc=e
+            )
+            return []
+
+        return response.json()[connector_name]["topics"]
+
+    def construct_flow_workunit(self, connector: ConnectorManifest) -> MetadataWorkUnit:
+        connector_name = connector.name
+        connector_type = connector.type
+        connector_class = connector.config.get(CONNECTOR_CLASS)
+        flow_property_bag = connector.flow_property_bag
+        # connector_url = connector.url  # NOTE: this will expose connector credential when used
+        flow_urn = builder.make_data_flow_urn(
+            self.platform,
+            connector_name,
+            self.config.env,
+            self.config.platform_instance,
+        )
+
+        return MetadataChangeProposalWrapper(
+            entityUrn=flow_urn,
+            aspect=models.DataFlowInfoClass(
+                name=connector_name,
+                description=f"{connector_type.capitalize()} connector using `{connector_class}` plugin.",
+                customProperties=flow_property_bag,
+                # externalUrl=connector_url, # NOTE: this will expose connector credential when used
+            ),
+        ).as_workunit()
+
+    def construct_job_workunits(
+        self, connector: ConnectorManifest
+    ) -> Iterable[MetadataWorkUnit]:
+        connector_name = connector.name
+        flow_urn = builder.make_data_flow_urn(
+            self.platform,
+            connector_name,
+            self.config.env,
+            self.config.platform_instance,
+        )
+
+        lineages = connector.lineages
+        if lineages:
+            for lineage in lineages:
+                source_dataset = lineage.source_dataset
+                source_platform = lineage.source_platform
+                target_dataset = lineage.target_dataset
+                target_platform = lineage.target_platform
+                job_property_bag = lineage.job_property_bag
+
+                source_platform_instance = get_platform_instance(
+                    self.config, connector_name, source_platform
+                )
+                target_platform_instance = get_platform_instance(
+                    self.config, connector_name, target_platform
+                )
+
+                job_id = self.get_job_id(lineage, connector, self.config)
+                job_urn = builder.make_data_job_urn_with_flow(flow_urn, job_id)
+
+                inlets = (
+                    [
+                        self.make_lineage_dataset_urn(
+                            source_platform, source_dataset, source_platform_instance
+                        )
+                    ]
+                    if source_dataset
+                    else []
+                )
+                outlets = [
+                    self.make_lineage_dataset_urn(
+                        target_platform, target_dataset, target_platform_instance
+                    )
+                ]
+
+                yield MetadataChangeProposalWrapper(
+                    entityUrn=job_urn,
+                    aspect=models.DataJobInfoClass(
+                        name=f"{connector_name}:{job_id}",
+                        type="COMMAND",
+                        customProperties=job_property_bag,
+                    ),
+                ).as_workunit()
+
+                yield MetadataChangeProposalWrapper(
+                    entityUrn=job_urn,
+                    aspect=models.DataJobInputOutputClass(
+                        inputDatasets=inlets,
+                        outputDatasets=outlets,
+                    ),
+                ).as_workunit()
+
+    def get_job_id(
+        self,
+        lineage: KafkaConnectLineage,
+        connector: ConnectorManifest,
+        config: KafkaConnectSourceConfig,
+    ) -> str:
+        connector_class = connector.config.get(CONNECTOR_CLASS)
+
+        # Note - This block is only to maintain backward compatibility of Job URN
+        if (
+            connector_class
+            and connector.type == SOURCE
+            and (
+                "JdbcSourceConnector" in connector_class
+                or connector_class.startswith("io.debezium.connector")
+            )
+            and lineage.source_dataset
+            and config.connect_to_platform_map
+            and config.connect_to_platform_map.get(connector.name)
+            and config.connect_to_platform_map[connector.name].get(
+                lineage.source_platform
+            )
+        ):
+            return f"{config.connect_to_platform_map[connector.name][lineage.source_platform]}.{lineage.source_dataset}"
+
+        return (
+            lineage.source_dataset
+            if lineage.source_dataset
+            else f"unknown_source.{lineage.target_dataset}"
+        )
+
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [
+            *super().get_workunit_processors(),
+            StaleEntityRemovalHandler.create(
+                self, self.config, self.ctx
+            ).workunit_processor,
+        ]
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        for connector in self.get_connectors_manifest():
+            yield self.construct_flow_workunit(connector)
+            yield from self.construct_job_workunits(connector)
+            self.report.report_connector_scanned(connector.name)
+
+    def get_report(self) -> KafkaConnectSourceReport:
+        return self.report
+
+    def make_lineage_dataset_urn(
+        self, platform: str, name: str, platform_instance: Optional[str]
+    ) -> str:
+        if self.config.convert_lineage_urns_to_lowercase:
+            name = name.lower()
+
+        return builder.make_dataset_urn_with_platform_instance(
+            platform, name, platform_instance, self.config.env
+        )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/sink_connectors.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/sink_connectors.py
new file mode 100644
index 0000000000000..2790460c8e601
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/sink_connectors.py
@@ -0,0 +1,341 @@
+import re
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from datahub.ingestion.source.kafka_connect.common import (
+    KAFKA,
+    BaseConnector,
+    ConnectorManifest,
+    KafkaConnectLineage,
+)
+
+
+@dataclass
+class ConfluentS3SinkConnector(BaseConnector):
+    @dataclass
+    class S3SinkParser:
+        target_platform: str
+        bucket: str
+        topics_dir: str
+        topics: Iterable[str]
+
+    def _get_parser(self, connector_manifest: ConnectorManifest) -> S3SinkParser:
+        # https://docs.confluent.io/kafka-connectors/s3-sink/current/configuration_options.html#s3
+        bucket = connector_manifest.config.get("s3.bucket.name")
+        if not bucket:
+            raise ValueError(
+                "Could not find 's3.bucket.name' in connector configuration"
+            )
+
+        # https://docs.confluent.io/kafka-connectors/s3-sink/current/configuration_options.html#storage
+        topics_dir = connector_manifest.config.get("topics.dir", "topics")
+
+        return self.S3SinkParser(
+            target_platform="s3",
+            bucket=bucket,
+            topics_dir=topics_dir,
+            topics=connector_manifest.topic_names,
+        )
+
+    def extract_flow_property_bag(self) -> Dict[str, str]:
+        # Mask/Remove properties that may reveal credentials
+        flow_property_bag = {
+            k: v
+            for k, v in self.connector_manifest.config.items()
+            if k
+            not in [
+                "aws.access.key.id",
+                "aws.secret.access.key",
+                "s3.sse.customer.key",
+                "s3.proxy.password",
+            ]
+        }
+        return flow_property_bag
+
+    def extract_lineages(self) -> List[KafkaConnectLineage]:
+        try:
+            parser = self._get_parser(self.connector_manifest)
+
+            lineages: List[KafkaConnectLineage] = list()
+            for topic in parser.topics:
+                target_dataset = f"{parser.bucket}/{parser.topics_dir}/{topic}"
+
+                lineages.append(
+                    KafkaConnectLineage(
+                        source_dataset=topic,
+                        source_platform="kafka",
+                        target_dataset=target_dataset,
+                        target_platform=parser.target_platform,
+                    )
+                )
+            return lineages
+        except Exception as e:
+            self.report.warning(
+                "Error resolving lineage for connector",
+                self.connector_manifest.name,
+                exc=e,
+            )
+
+        return []
+
+
+@dataclass
+class SnowflakeSinkConnector(BaseConnector):
+    @dataclass
+    class SnowflakeParser:
+        database_name: str
+        schema_name: str
+        topics_to_tables: Dict[str, str]
+
+    def get_table_name_from_topic_name(self, topic_name: str) -> str:
+        """
+        This function converts the topic name to a valid Snowflake table name using some rules.
+        Refer below link for more info
+        https://docs.snowflake.com/en/user-guide/kafka-connector-overview#target-tables-for-kafka-topics
+        """
+        table_name = re.sub("[^a-zA-Z0-9_]", "_", topic_name)
+        if re.match("^[^a-zA-Z_].*", table_name):
+            table_name = "_" + table_name
+        # Connector  may append original topic's hash code as suffix for conflict resolution
+        # if generated table names for 2 topics are similar. This corner case is not handled here.
+        # Note that Snowflake recommends to choose topic names that follow the rules for
+        # Snowflake identifier names so this case is not recommended by snowflake.
+        return table_name
+
+    def get_parser(
+        self,
+        connector_manifest: ConnectorManifest,
+    ) -> SnowflakeParser:
+        database_name = connector_manifest.config["snowflake.database.name"]
+        schema_name = connector_manifest.config["snowflake.schema.name"]
+
+        # Fetch user provided topic to table map
+        provided_topics_to_tables: Dict[str, str] = {}
+        if connector_manifest.config.get("snowflake.topic2table.map"):
+            for each in connector_manifest.config["snowflake.topic2table.map"].split(
+                ","
+            ):
+                topic, table = each.split(":")
+                provided_topics_to_tables[topic.strip()] = table.strip()
+
+        topics_to_tables: Dict[str, str] = {}
+        # Extract lineage for only those topics whose data ingestion started
+        for topic in connector_manifest.topic_names:
+            if topic in provided_topics_to_tables:
+                # If user provided which table to get mapped with this topic
+                topics_to_tables[topic] = provided_topics_to_tables[topic]
+            else:
+                # Else connector converts topic name to a valid Snowflake table name.
+                topics_to_tables[topic] = self.get_table_name_from_topic_name(topic)
+
+        return self.SnowflakeParser(
+            database_name=database_name,
+            schema_name=schema_name,
+            topics_to_tables=topics_to_tables,
+        )
+
+    def extract_flow_property_bag(self) -> Dict[str, str]:
+        # For all snowflake sink connector properties, refer below link
+        # https://docs.snowflake.com/en/user-guide/kafka-connector-install#configuring-the-kafka-connector
+        # remove private keys, secrets from properties
+        flow_property_bag = {
+            k: v
+            for k, v in self.connector_manifest.config.items()
+            if k
+            not in [
+                "snowflake.private.key",
+                "snowflake.private.key.passphrase",
+                "value.converter.basic.auth.user.info",
+            ]
+        }
+
+        return flow_property_bag
+
+    def extract_lineages(self) -> List[KafkaConnectLineage]:
+        lineages: List[KafkaConnectLineage] = list()
+        parser = self.get_parser(self.connector_manifest)
+
+        for topic, table in parser.topics_to_tables.items():
+            target_dataset = f"{parser.database_name}.{parser.schema_name}.{table}"
+            lineages.append(
+                KafkaConnectLineage(
+                    source_dataset=topic,
+                    source_platform=KAFKA,
+                    target_dataset=target_dataset,
+                    target_platform="snowflake",
+                )
+            )
+
+        return lineages
+
+
+@dataclass
+class BigQuerySinkConnector(BaseConnector):
+    @dataclass
+    class BQParser:
+        project: str
+        target_platform: str
+        sanitizeTopics: str
+        transforms: list
+        topicsToTables: Optional[str] = None
+        datasets: Optional[str] = None
+        defaultDataset: Optional[str] = None
+        version: str = "v1"
+
+    def get_parser(
+        self,
+        connector_manifest: ConnectorManifest,
+    ) -> BQParser:
+        project = connector_manifest.config["project"]
+        sanitizeTopics = connector_manifest.config.get("sanitizeTopics", "false")
+        transform_names = (
+            self.connector_manifest.config.get("transforms", "").split(",")
+            if self.connector_manifest.config.get("transforms")
+            else []
+        )
+        transforms = []
+        for name in transform_names:
+            transform = {"name": name}
+            transforms.append(transform)
+            for key in self.connector_manifest.config.keys():
+                if key.startswith(f"transforms.{name}."):
+                    transform[
+                        key.replace(f"transforms.{name}.", "")
+                    ] = self.connector_manifest.config[key]
+
+        if "defaultDataset" in connector_manifest.config:
+            defaultDataset = connector_manifest.config["defaultDataset"]
+            return self.BQParser(
+                project=project,
+                defaultDataset=defaultDataset,
+                target_platform="bigquery",
+                sanitizeTopics=sanitizeTopics.lower() == "true",
+                version="v2",
+                transforms=transforms,
+            )
+        else:
+            # version 1.6.x and similar configs supported
+            datasets = connector_manifest.config["datasets"]
+            topicsToTables = connector_manifest.config.get("topicsToTables")
+
+            return self.BQParser(
+                project=project,
+                topicsToTables=topicsToTables,
+                datasets=datasets,
+                target_platform="bigquery",
+                sanitizeTopics=sanitizeTopics.lower() == "true",
+                transforms=transforms,
+            )
+
+    def get_list(self, property: str) -> Iterable[Tuple[str, str]]:
+        entries = property.split(",")
+        for entry in entries:
+            key, val = entry.rsplit("=")
+            yield (key.strip(), val.strip())
+
+    def get_dataset_for_topic_v1(self, topic: str, parser: BQParser) -> Optional[str]:
+        topicregex_dataset_map: Dict[str, str] = dict(self.get_list(parser.datasets))  # type: ignore
+        from java.util.regex import Pattern
+
+        for pattern, dataset in topicregex_dataset_map.items():
+            patternMatcher = Pattern.compile(pattern).matcher(topic)
+            if patternMatcher.matches():
+                return dataset
+        return None
+
+    def sanitize_table_name(self, table_name):
+        table_name = re.sub("[^a-zA-Z0-9_]", "_", table_name)
+        if re.match("^[^a-zA-Z_].*", table_name):
+            table_name = "_" + table_name
+
+        return table_name
+
+    def get_dataset_table_for_topic(
+        self, topic: str, parser: BQParser
+    ) -> Optional[str]:
+        if parser.version == "v2":
+            dataset = parser.defaultDataset
+            parts = topic.split(":")
+            if len(parts) == 2:
+                dataset = parts[0]
+                table = parts[1]
+            else:
+                table = parts[0]
+        else:
+            dataset = self.get_dataset_for_topic_v1(topic, parser)
+            if dataset is None:
+                return None
+
+            table = topic
+            if parser.topicsToTables:
+                topicregex_table_map: Dict[str, str] = dict(
+                    self.get_list(parser.topicsToTables)  # type: ignore
+                )
+                from java.util.regex import Pattern
+
+                for pattern, tbl in topicregex_table_map.items():
+                    patternMatcher = Pattern.compile(pattern).matcher(topic)
+                    if patternMatcher.matches():
+                        table = tbl
+                        break
+
+        if parser.sanitizeTopics:
+            table = self.sanitize_table_name(table)
+        return f"{dataset}.{table}"
+
+    def apply_transformations(
+        self, topic: str, transforms: List[Dict[str, str]]
+    ) -> str:
+        for transform in transforms:
+            if transform["type"] == "org.apache.kafka.connect.transforms.RegexRouter":
+                regex = transform["regex"]
+                replacement = transform["replacement"]
+                pattern = re.compile(regex)
+                if pattern.match(topic):
+                    topic = pattern.sub(replacement, topic, count=1)
+        return topic
+
+    def extract_flow_property_bag(self) -> Dict[str, str]:
+        # Mask/Remove properties that may reveal credentials
+        flow_property_bag = {
+            k: v
+            for k, v in self.connector_manifest.config.items()
+            if k not in ["keyfile"]
+        }
+
+        return flow_property_bag
+
+    def extract_lineages(self) -> List[KafkaConnectLineage]:
+        lineages: List[KafkaConnectLineage] = list()
+        parser = self.get_parser(self.connector_manifest)
+        if not parser:
+            return lineages
+        target_platform = parser.target_platform
+        project = parser.project
+        transforms = parser.transforms
+
+        for topic in self.connector_manifest.topic_names:
+            transformed_topic = self.apply_transformations(topic, transforms)
+            dataset_table = self.get_dataset_table_for_topic(transformed_topic, parser)
+            if dataset_table is None:
+                self.report.warning(
+                    "Could not find target dataset for topic, please check your connector configuration"
+                    f"{self.connector_manifest.name} : {transformed_topic} ",
+                )
+                continue
+            target_dataset = f"{project}.{dataset_table}"
+
+            lineages.append(
+                KafkaConnectLineage(
+                    source_dataset=transformed_topic,
+                    source_platform=KAFKA,
+                    target_dataset=target_dataset,
+                    target_platform=target_platform,
+                )
+            )
+        return lineages
+
+
+BIGQUERY_SINK_CONNECTOR_CLASS = "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector"
+S3_SINK_CONNECTOR_CLASS = "io.confluent.connect.s3.S3SinkConnector"
+SNOWFLAKE_SINK_CONNECTOR_CLASS = "com.snowflake.kafka.connector.SnowflakeSinkConnector"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/source_connectors.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/source_connectors.py
new file mode 100644
index 0000000000000..7b3b6e551a0a1
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/source_connectors.py
@@ -0,0 +1,570 @@
+import logging
+import re
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from sqlalchemy.engine.url import make_url
+
+from datahub.ingestion.source.kafka_connect.common import (
+    CONNECTOR_CLASS,
+    KAFKA,
+    BaseConnector,
+    ConnectorManifest,
+    KafkaConnectLineage,
+    get_dataset_name,
+    has_three_level_hierarchy,
+    remove_prefix,
+    unquote,
+)
+from datahub.ingestion.source.sql.sqlalchemy_uri_mapper import (
+    get_platform_from_sqlalchemy_uri,
+)
+
+
+@dataclass
+class ConfluentJDBCSourceConnector(BaseConnector):
+    REGEXROUTER = "org.apache.kafka.connect.transforms.RegexRouter"
+    KNOWN_TOPICROUTING_TRANSFORMS = [REGEXROUTER]
+    # https://kafka.apache.org/documentation/#connect_included_transformation
+    KAFKA_NONTOPICROUTING_TRANSFORMS = [
+        "InsertField",
+        "InsertField$Key",
+        "InsertField$Value",
+        "ReplaceField",
+        "ReplaceField$Key",
+        "ReplaceField$Value",
+        "MaskField",
+        "MaskField$Key",
+        "MaskField$Value",
+        "ValueToKey",
+        "ValueToKey$Key",
+        "ValueToKey$Value",
+        "HoistField",
+        "HoistField$Key",
+        "HoistField$Value",
+        "ExtractField",
+        "ExtractField$Key",
+        "ExtractField$Value",
+        "SetSchemaMetadata",
+        "SetSchemaMetadata$Key",
+        "SetSchemaMetadata$Value",
+        "Flatten",
+        "Flatten$Key",
+        "Flatten$Value",
+        "Cast",
+        "Cast$Key",
+        "Cast$Value",
+        "HeadersFrom",
+        "HeadersFrom$Key",
+        "HeadersFrom$Value",
+        "TimestampConverter",
+        "Filter",
+        "InsertHeader",
+        "DropHeaders",
+    ]
+    # https://docs.confluent.io/platform/current/connect/transforms/overview.html
+    CONFLUENT_NONTOPICROUTING_TRANSFORMS = [
+        "Drop",
+        "Drop$Key",
+        "Drop$Value",
+        "Filter",
+        "Filter$Key",
+        "Filter$Value",
+        "TombstoneHandler",
+    ]
+    KNOWN_NONTOPICROUTING_TRANSFORMS = (
+        KAFKA_NONTOPICROUTING_TRANSFORMS
+        + [
+            f"org.apache.kafka.connect.transforms.{t}"
+            for t in KAFKA_NONTOPICROUTING_TRANSFORMS
+        ]
+        + CONFLUENT_NONTOPICROUTING_TRANSFORMS
+        + [
+            f"io.confluent.connect.transforms.{t}"
+            for t in CONFLUENT_NONTOPICROUTING_TRANSFORMS
+        ]
+    )
+
+    @dataclass
+    class JdbcParser:
+        db_connection_url: str
+        source_platform: str
+        database_name: str
+        topic_prefix: str
+        query: str
+        transforms: list
+
+    def get_parser(
+        self,
+        connector_manifest: ConnectorManifest,
+    ) -> JdbcParser:
+        url = remove_prefix(
+            str(connector_manifest.config.get("connection.url")), "jdbc:"
+        )
+        url_instance = make_url(url)
+        source_platform = get_platform_from_sqlalchemy_uri(str(url_instance))
+        database_name = url_instance.database
+        assert database_name
+        db_connection_url = f"{url_instance.drivername}://{url_instance.host}:{url_instance.port}/{database_name}"
+
+        topic_prefix = self.connector_manifest.config.get("topic.prefix", None)
+
+        query = self.connector_manifest.config.get("query", None)
+
+        transform_names = (
+            self.connector_manifest.config.get("transforms", "").split(",")
+            if self.connector_manifest.config.get("transforms")
+            else []
+        )
+
+        transforms = []
+        for name in transform_names:
+            transform = {"name": name}
+            transforms.append(transform)
+            for key in self.connector_manifest.config.keys():
+                if key.startswith(f"transforms.{name}."):
+                    transform[
+                        key.replace(f"transforms.{name}.", "")
+                    ] = self.connector_manifest.config[key]
+
+        return self.JdbcParser(
+            db_connection_url,
+            source_platform,
+            database_name,
+            topic_prefix,
+            query,
+            transforms,
+        )
+
+    def default_get_lineages(
+        self,
+        topic_prefix: str,
+        database_name: str,
+        source_platform: str,
+        topic_names: Optional[Iterable[str]] = None,
+        include_source_dataset: bool = True,
+    ) -> List[KafkaConnectLineage]:
+        lineages: List[KafkaConnectLineage] = []
+        if not topic_names:
+            topic_names = self.connector_manifest.topic_names
+        table_name_tuples: List[Tuple] = self.get_table_names()
+        for topic in topic_names:
+            # All good for NO_TRANSFORM or (SINGLE_TRANSFORM and KNOWN_NONTOPICROUTING_TRANSFORM) or (not SINGLE_TRANSFORM and all(KNOWN_NONTOPICROUTING_TRANSFORM))
+            source_table: str = (
+                remove_prefix(topic, topic_prefix) if topic_prefix else topic
+            )
+            # include schema name for three-level hierarchies
+            if has_three_level_hierarchy(source_platform):
+                table_name_tuple: Tuple = next(
+                    iter([t for t in table_name_tuples if t and t[-1] == source_table]),
+                    (),
+                )
+                if len(table_name_tuple) > 1:
+                    source_table = f"{table_name_tuple[-2]}.{source_table}"
+                else:
+                    include_source_dataset = False
+                    self.report.warning(
+                        "Could not find schema for table"
+                        f"{self.connector_manifest.name} : {source_table}",
+                    )
+            dataset_name: str = get_dataset_name(database_name, source_table)
+            lineage = KafkaConnectLineage(
+                source_dataset=dataset_name if include_source_dataset else None,
+                source_platform=source_platform,
+                target_dataset=topic,
+                target_platform=KAFKA,
+            )
+            lineages.append(lineage)
+        return lineages
+
+    def get_table_names(self) -> List[Tuple]:
+        sep: str = "."
+        leading_quote_char: str = '"'
+        trailing_quote_char: str = leading_quote_char
+
+        table_ids: List[str] = []
+        if self.connector_manifest.tasks:
+            table_ids = (
+                ",".join(
+                    [
+                        task["config"].get("tables")
+                        for task in self.connector_manifest.tasks
+                    ]
+                )
+            ).split(",")
+            quote_method = self.connector_manifest.config.get(
+                "quote.sql.identifiers", "always"
+            )
+            if (
+                quote_method == "always"
+                and table_ids
+                and table_ids[0]
+                and table_ids[-1]
+            ):
+                leading_quote_char = table_ids[0][0]
+                trailing_quote_char = table_ids[-1][-1]
+                # This will only work for single character quotes
+        elif self.connector_manifest.config.get("table.whitelist"):
+            table_ids = self.connector_manifest.config.get("table.whitelist").split(",")  # type: ignore
+
+        # List of Tuple containing (schema, table)
+        tables: List[Tuple] = [
+            (
+                (
+                    unquote(
+                        table_id.split(sep)[-2], leading_quote_char, trailing_quote_char
+                    )
+                    if len(table_id.split(sep)) > 1
+                    else ""
+                ),
+                unquote(
+                    table_id.split(sep)[-1], leading_quote_char, trailing_quote_char
+                ),
+            )
+            for table_id in table_ids
+        ]
+        return tables
+
+    def extract_flow_property_bag(self) -> Dict[str, str]:
+        flow_property_bag = {
+            k: v
+            for k, v in self.connector_manifest.config.items()
+            if k not in ["connection.password", "connection.user"]
+        }
+
+        # Mask/Remove properties that may reveal credentials
+        flow_property_bag["connection.url"] = self.get_parser(
+            self.connector_manifest
+        ).db_connection_url
+
+        return flow_property_bag
+
+    def extract_lineages(self) -> List[KafkaConnectLineage]:
+        lineages: List[KafkaConnectLineage] = list()
+        parser = self.get_parser(self.connector_manifest)
+        source_platform = parser.source_platform
+        database_name = parser.database_name
+        query = parser.query
+        topic_prefix = parser.topic_prefix
+        transforms = parser.transforms
+
+        logging.debug(
+            f"Extracting source platform: {source_platform} and database name: {database_name} from connection url "
+        )
+
+        if not self.connector_manifest.topic_names:
+            return lineages
+
+        if query:
+            # Lineage source_table can be extracted by parsing query
+            for topic in self.connector_manifest.topic_names:
+                # default method - as per earlier implementation
+                dataset_name: str = get_dataset_name(database_name, topic)
+
+                lineage = KafkaConnectLineage(
+                    source_dataset=None,
+                    source_platform=source_platform,
+                    target_dataset=topic,
+                    target_platform=KAFKA,
+                )
+                lineages.append(lineage)
+                self.report.warning(
+                    "Could not find input dataset, the connector has query configuration set",
+                    self.connector_manifest.name,
+                )
+                return lineages
+
+        SINGLE_TRANSFORM = len(transforms) == 1
+        NO_TRANSFORM = len(transforms) == 0
+        UNKNOWN_TRANSFORM = any(
+            [
+                transform["type"]
+                not in self.KNOWN_TOPICROUTING_TRANSFORMS
+                + self.KNOWN_NONTOPICROUTING_TRANSFORMS
+                for transform in transforms
+            ]
+        )
+        ALL_TRANSFORMS_NON_TOPICROUTING = all(
+            [
+                transform["type"] in self.KNOWN_NONTOPICROUTING_TRANSFORMS
+                for transform in transforms
+            ]
+        )
+
+        if NO_TRANSFORM or ALL_TRANSFORMS_NON_TOPICROUTING:
+            return self.default_get_lineages(
+                database_name=database_name,
+                source_platform=source_platform,
+                topic_prefix=topic_prefix,
+            )
+
+        if SINGLE_TRANSFORM and transforms[0]["type"] == self.REGEXROUTER:
+            tables = self.get_table_names()
+            topic_names = list(self.connector_manifest.topic_names)
+
+            from java.util.regex import Pattern
+
+            for table in tables:
+                source_table: str = table[-1]
+                topic = topic_prefix + source_table if topic_prefix else source_table
+
+                transform_regex = Pattern.compile(transforms[0]["regex"])
+                transform_replacement = transforms[0]["replacement"]
+
+                matcher = transform_regex.matcher(topic)
+                if matcher.matches():
+                    topic = str(matcher.replaceFirst(transform_replacement))
+
+                # Additional check to confirm that the topic present
+                # in connector topics
+
+                if topic in self.connector_manifest.topic_names:
+                    # include schema name for three-level hierarchies
+                    if has_three_level_hierarchy(source_platform) and len(table) > 1:
+                        source_table = f"{table[-2]}.{table[-1]}"
+
+                    dataset_name = get_dataset_name(database_name, source_table)
+
+                    lineage = KafkaConnectLineage(
+                        source_dataset=dataset_name,
+                        source_platform=source_platform,
+                        target_dataset=topic,
+                        target_platform=KAFKA,
+                    )
+                    topic_names.remove(topic)
+                    lineages.append(lineage)
+
+            if topic_names:
+                lineages.extend(
+                    self.default_get_lineages(
+                        database_name=database_name,
+                        source_platform=source_platform,
+                        topic_prefix=topic_prefix,
+                        topic_names=topic_names,
+                        include_source_dataset=False,
+                    )
+                )
+                self.report.warning(
+                    "Could not find input dataset for connector topics",
+                    f"{self.connector_manifest.name} : {topic_names}",
+                )
+            return lineages
+        else:
+            include_source_dataset = True
+            if SINGLE_TRANSFORM and UNKNOWN_TRANSFORM:
+                self.report.warning(
+                    "Could not find input dataset, connector has unknown transform",
+                    f"{self.connector_manifest.name} : {transforms[0]['type']}",
+                )
+                include_source_dataset = False
+            if not SINGLE_TRANSFORM and UNKNOWN_TRANSFORM:
+                self.report.warning(
+                    "Could not find input dataset, connector has one or more unknown transforms",
+                    self.connector_manifest.name,
+                )
+                include_source_dataset = False
+            lineages = self.default_get_lineages(
+                database_name=database_name,
+                source_platform=source_platform,
+                topic_prefix=topic_prefix,
+                include_source_dataset=include_source_dataset,
+            )
+            return lineages
+
+
+@dataclass
+class MongoSourceConnector(BaseConnector):
+    # https://www.mongodb.com/docs/kafka-connector/current/source-connector/
+
+    @dataclass
+    class MongoSourceParser:
+        db_connection_url: Optional[str]
+        source_platform: str
+        database_name: Optional[str]
+        topic_prefix: Optional[str]
+        transforms: List[str]
+
+    def get_parser(
+        self,
+        connector_manifest: ConnectorManifest,
+    ) -> MongoSourceParser:
+        parser = self.MongoSourceParser(
+            db_connection_url=connector_manifest.config.get("connection.uri"),
+            source_platform="mongodb",
+            database_name=connector_manifest.config.get("database"),
+            topic_prefix=connector_manifest.config.get("topic_prefix"),
+            transforms=(
+                connector_manifest.config["transforms"].split(",")
+                if "transforms" in connector_manifest.config
+                else []
+            ),
+        )
+
+        return parser
+
+    def extract_lineages(self) -> List[KafkaConnectLineage]:
+        lineages: List[KafkaConnectLineage] = list()
+        parser = self.get_parser(self.connector_manifest)
+        source_platform = parser.source_platform
+        topic_naming_pattern = r"mongodb\.(\w+)\.(\w+)"
+
+        if not self.connector_manifest.topic_names:
+            return lineages
+
+        for topic in self.connector_manifest.topic_names:
+            found = re.search(re.compile(topic_naming_pattern), topic)
+
+            if found:
+                table_name = get_dataset_name(found.group(1), found.group(2))
+
+                lineage = KafkaConnectLineage(
+                    source_dataset=table_name,
+                    source_platform=source_platform,
+                    target_dataset=topic,
+                    target_platform=KAFKA,
+                )
+                lineages.append(lineage)
+        return lineages
+
+
+@dataclass
+class DebeziumSourceConnector(BaseConnector):
+    @dataclass
+    class DebeziumParser:
+        source_platform: str
+        server_name: Optional[str]
+        database_name: Optional[str]
+
+    def get_server_name(self, connector_manifest: ConnectorManifest) -> str:
+        if "topic.prefix" in connector_manifest.config:
+            return connector_manifest.config["topic.prefix"]
+        else:
+            return connector_manifest.config.get("database.server.name", "")
+
+    def get_parser(
+        self,
+        connector_manifest: ConnectorManifest,
+    ) -> DebeziumParser:
+        connector_class = connector_manifest.config.get(CONNECTOR_CLASS, "")
+
+        if connector_class == "io.debezium.connector.mysql.MySqlConnector":
+            parser = self.DebeziumParser(
+                source_platform="mysql",
+                server_name=self.get_server_name(connector_manifest),
+                database_name=None,
+            )
+        elif connector_class == "MySqlConnector":
+            parser = self.DebeziumParser(
+                source_platform="mysql",
+                server_name=self.get_server_name(connector_manifest),
+                database_name=None,
+            )
+        elif connector_class == "io.debezium.connector.mongodb.MongoDbConnector":
+            parser = self.DebeziumParser(
+                source_platform="mongodb",
+                server_name=self.get_server_name(connector_manifest),
+                database_name=None,
+            )
+        elif connector_class == "io.debezium.connector.postgresql.PostgresConnector":
+            parser = self.DebeziumParser(
+                source_platform="postgres",
+                server_name=self.get_server_name(connector_manifest),
+                database_name=connector_manifest.config.get("database.dbname"),
+            )
+        elif connector_class == "io.debezium.connector.oracle.OracleConnector":
+            parser = self.DebeziumParser(
+                source_platform="oracle",
+                server_name=self.get_server_name(connector_manifest),
+                database_name=connector_manifest.config.get("database.dbname"),
+            )
+        elif connector_class == "io.debezium.connector.sqlserver.SqlServerConnector":
+            database_name = connector_manifest.config.get(
+                "database.names"
+            ) or connector_manifest.config.get("database.dbname")
+
+            if "," in str(database_name):
+                raise Exception(
+                    f"Only one database is supported for Debezium's SQL Server connector. Found: {database_name}"
+                )
+
+            parser = self.DebeziumParser(
+                source_platform="mssql",
+                server_name=self.get_server_name(connector_manifest),
+                database_name=database_name,
+            )
+        elif connector_class == "io.debezium.connector.db2.Db2Connector":
+            parser = self.DebeziumParser(
+                source_platform="db2",
+                server_name=self.get_server_name(connector_manifest),
+                database_name=connector_manifest.config.get("database.dbname"),
+            )
+        elif connector_class == "io.debezium.connector.vitess.VitessConnector":
+            parser = self.DebeziumParser(
+                source_platform="vitess",
+                server_name=self.get_server_name(connector_manifest),
+                database_name=connector_manifest.config.get("vitess.keyspace"),
+            )
+        else:
+            raise ValueError(f"Connector class '{connector_class}' is unknown.")
+
+        return parser
+
+    def extract_lineages(self) -> List[KafkaConnectLineage]:
+        lineages: List[KafkaConnectLineage] = list()
+
+        try:
+            parser = self.get_parser(self.connector_manifest)
+            source_platform = parser.source_platform
+            server_name = parser.server_name
+            database_name = parser.database_name
+            topic_naming_pattern = rf"({server_name})\.(\w+\.\w+)"
+
+            if not self.connector_manifest.topic_names:
+                return lineages
+
+            for topic in self.connector_manifest.topic_names:
+                found = re.search(re.compile(topic_naming_pattern), topic)
+
+                if found:
+                    table_name = get_dataset_name(database_name, found.group(2))
+
+                    lineage = KafkaConnectLineage(
+                        source_dataset=table_name,
+                        source_platform=source_platform,
+                        target_dataset=topic,
+                        target_platform=KAFKA,
+                    )
+                    lineages.append(lineage)
+            return lineages
+        except Exception as e:
+            self.report.warning(
+                "Error resolving lineage for connector",
+                self.connector_manifest.name,
+                exc=e,
+            )
+
+        return []
+
+
+@dataclass
+class ConfigDrivenSourceConnector(BaseConnector):
+    def extract_lineages(self) -> List[KafkaConnectLineage]:
+        lineages = []
+        for connector in self.config.generic_connectors:
+            if connector.connector_name == self.connector_manifest.name:
+                target_connector = connector
+                break
+        for topic in self.connector_manifest.topic_names:
+            lineage = KafkaConnectLineage(
+                source_dataset=target_connector.source_dataset,
+                source_platform=target_connector.source_platform,
+                target_dataset=topic,
+                target_platform=KAFKA,
+            )
+            lineages.append(lineage)
+        return lineages
+
+
+JDBC_SOURCE_CONNECTOR_CLASS = "io.confluent.connect.jdbc.JdbcSourceConnector"
+DEBEZIUM_SOURCE_CONNECTOR_PREFIX = "io.debezium.connector"
+MONGO_SOURCE_CONNECTOR_CLASS = "com.mongodb.kafka.connect.MongoSourceConnector"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
index 3cbb13375229b..1183916e9b3fe 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
@@ -31,6 +31,10 @@
 from pydantic.class_validators import validator
 
 import datahub.emitter.mce_builder as builder
+from datahub.api.entities.platformresource.platform_resource import (
+    PlatformResource,
+    PlatformResourceKey,
+)
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import ContainerKey, create_embed_mcp
 from datahub.ingestion.api.report import Report
@@ -48,7 +52,7 @@
 from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader
 from datahub.ingestion.source.looker.looker_lib_wrapper import LookerAPI
 from datahub.ingestion.source.looker.lookml_config import (
-    _BASE_PROJECT_NAME,
+    BASE_PROJECT_NAME,
     LookMLSourceReport,
 )
 from datahub.ingestion.source.looker.str_functions import remove_suffix
@@ -106,7 +110,7 @@
 from datahub.utilities.url_util import remove_port_from_url
 
 CORPUSER_DATAHUB = "urn:li:corpuser:datahub"
-
+LOOKER = "looker"
 logger = logging.getLogger(__name__)
 
 
@@ -307,7 +311,6 @@ def view_fields_from_dict(
         type_cls: ViewFieldType,
         populate_sql_logic_in_descriptions: bool,
     ) -> "ViewField":
-
         is_primary_key = field_dict.get("primary_key", "no") == "yes"
 
         name = field_dict["name"]
@@ -370,7 +373,7 @@ def _form_field_name(
         assert view_name  # for lint false positive
 
         project_include: ProjectInclude = ProjectInclude(
-            project=view_project_map.get(view_name, _BASE_PROJECT_NAME),
+            project=view_project_map.get(view_name, BASE_PROJECT_NAME),
             include=view_name,
         )
 
@@ -385,7 +388,7 @@ def _form_field_name(
         view_urn = LookerViewId(
             project_name=(
                 project_include.project
-                if project_include.project != _BASE_PROJECT_NAME
+                if project_include.project != BASE_PROJECT_NAME
                 else explore_project_name
             ),
             model_name=model_name,
@@ -929,8 +932,6 @@ def from_api(  # noqa: C901
         reporter: SourceReport,
         source_config: LookerDashboardSourceConfig,
     ) -> Optional["LookerExplore"]:  # noqa: C901
-        from datahub.ingestion.source.looker.lookml_source import _BASE_PROJECT_NAME
-
         try:
             explore = client.lookml_model_explore(model, explore_name)
             views: Set[str] = set()
@@ -988,13 +989,11 @@ def from_api(  # noqa: C901
             field_name_vs_raw_explore_field: Dict = {}
 
             if explore.fields is not None:
-
                 if explore.fields.dimensions is not None:
                     for dim_field in explore.fields.dimensions:
                         if dim_field.name is None:
                             continue
                         else:
-
                             field_name_vs_raw_explore_field[dim_field.name] = dim_field
 
                             view_fields.append(
@@ -1035,7 +1034,6 @@ def from_api(  # noqa: C901
                         if measure_field.name is None:
                             continue
                         else:
-
                             field_name_vs_raw_explore_field[
                                 measure_field.name
                             ] = measure_field
@@ -1114,7 +1112,7 @@ def from_api(  # noqa: C901
                 fields=view_fields,
                 upstream_views=list(
                     ProjectInclude(
-                        project=view_project_map.get(view_name, _BASE_PROJECT_NAME),
+                        project=view_project_map.get(view_name, BASE_PROJECT_NAME),
                         include=view_name,
                     )
                     for view_name in views
@@ -1194,7 +1192,6 @@ def _to_metadata_events(  # noqa: C901
     ) -> Optional[List[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]]]:
         # We only generate MCE-s for explores that contain from clauses and do NOT contain joins
         # All other explores (passthrough explores and joins) end in correct resolution of lineage, and don't need additional nodes in the graph.
-        from datahub.ingestion.source.looker.lookml_source import _BASE_PROJECT_NAME
 
         dataset_snapshot = DatasetSnapshot(
             urn=self.get_explore_urn(config),
@@ -1207,15 +1204,19 @@ def _to_metadata_events(  # noqa: C901
         dataset_snapshot.aspects.append(browse_paths)
         dataset_snapshot.aspects.append(StatusClass(removed=False))
 
-        custom_properties = {}
-        if self.label is not None:
-            custom_properties["looker.explore.label"] = str(self.label)
-        if self.source_file is not None:
-            custom_properties["looker.explore.file"] = str(self.source_file)
+        custom_properties = {
+            "project": self.project_name,
+            "model": self.model_name,
+            "looker.explore.label": self.label,
+            "looker.explore.name": self.name,
+            "looker.explore.file": self.source_file,
+        }
         dataset_props = DatasetPropertiesClass(
             name=str(self.label) if self.label else LookerUtil._display_name(self.name),
             description=self.description,
-            customProperties=custom_properties,
+            customProperties={
+                k: str(v) for k, v in custom_properties.items() if v is not None
+            },
         )
         dataset_props.externalUrl = self._get_url(base_url)
 
@@ -1237,7 +1238,7 @@ def _to_metadata_events(  # noqa: C901
                 view_urn = LookerViewId(
                     project_name=(
                         view_ref.project
-                        if view_ref.project != _BASE_PROJECT_NAME
+                        if view_ref.project != BASE_PROJECT_NAME
                         else self.project_name
                     ),
                     model_name=self.model_name,
@@ -1407,6 +1408,15 @@ class LookerDashboardSourceReport(StaleEntityRemovalSourceReport):
     dashboards_with_activity: LossySet[str] = dataclasses_field(
         default_factory=LossySet
     )
+
+    # Entities that don't seem to exist, so we don't emit usage aspects for them despite having usage data
+    dashboards_skipped_for_usage: LossySet[str] = dataclasses_field(
+        default_factory=LossySet
+    )
+    charts_skipped_for_usage: LossySet[str] = dataclasses_field(
+        default_factory=LossySet
+    )
+
     stage_latency: List[StageLatency] = dataclasses_field(default_factory=list)
     _looker_explore_registry: Optional[LookerExploreRegistry] = None
     total_explores: int = 0
@@ -1414,6 +1424,7 @@ class LookerDashboardSourceReport(StaleEntityRemovalSourceReport):
 
     resolved_user_ids: int = 0
     email_ids_missing: int = 0  # resolved users with missing email addresses
+    looker_user_count: int = 0
 
     _looker_api: Optional[LookerAPI] = None
     query_latency: Dict[str, datetime.timedelta] = dataclasses_field(
@@ -1617,9 +1628,21 @@ def get_urn_dashboard_id(self):
 class LookerUserRegistry:
     looker_api_wrapper: LookerAPI
     fields: str = ",".join(["id", "email", "display_name", "first_name", "last_name"])
+    _user_cache: Dict[str, LookerUser] = {}
 
-    def __init__(self, looker_api: LookerAPI):
+    def __init__(self, looker_api: LookerAPI, report: LookerDashboardSourceReport):
         self.looker_api_wrapper = looker_api
+        self.report = report
+        self._initialize_user_cache()
+
+    def _initialize_user_cache(self) -> None:
+        raw_users: Sequence[User] = self.looker_api_wrapper.all_users(
+            user_fields=self.fields
+        )
+
+        for raw_user in raw_users:
+            looker_user = LookerUser.create_looker_user(raw_user)
+            self._user_cache[str(looker_user.id)] = looker_user
 
     def get_by_id(self, id_: str) -> Optional[LookerUser]:
         if not id_:
@@ -1627,6 +1650,9 @@ def get_by_id(self, id_: str) -> Optional[LookerUser]:
 
         logger.debug(f"Will get user {id_}")
 
+        if str(id_) in self._user_cache:
+            return self._user_cache.get(str(id_))
+
         raw_user: Optional[User] = self.looker_api_wrapper.get_user(
             str(id_), user_fields=self.fields
         )
@@ -1635,3 +1661,35 @@ def get_by_id(self, id_: str) -> Optional[LookerUser]:
 
         looker_user = LookerUser.create_looker_user(raw_user)
         return looker_user
+
+    def to_platform_resource(
+        self, platform_instance: Optional[str]
+    ) -> Iterable[MetadataChangeProposalWrapper]:
+        try:
+            platform_resource_key = PlatformResourceKey(
+                platform=LOOKER,
+                resource_type="USER_ID_MAPPING",
+                platform_instance=platform_instance,
+                primary_key="",
+            )
+
+            # Extract user email mappings
+            user_email_cache = {
+                user_id: user.email
+                for user_id, user in self._user_cache.items()
+                if user.email
+            }
+
+            platform_resource = PlatformResource.create(
+                key=platform_resource_key,
+                value=user_email_cache,
+            )
+
+            self.report.looker_user_count = len(user_email_cache)
+            yield from platform_resource.to_mcps()
+
+        except Exception as exc:
+            self.report.warning(
+                message="Failed to generate platform resource for looker id mappings",
+                exc=exc,
+            )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py
index 7e23079156b62..327c9ebf99bd2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py
@@ -9,8 +9,8 @@
     load_and_preprocess_file,
 )
 from datahub.ingestion.source.looker.lookml_config import (
-    _BASE_PROJECT_NAME,
-    _EXPLORE_FILE_EXTENSION,
+    BASE_PROJECT_NAME,
+    EXPLORE_FILE_EXTENSION,
     LookMLSourceConfig,
     LookMLSourceReport,
 )
@@ -69,7 +69,7 @@ def from_looker_dict(
         explore_files = [
             x.include
             for x in resolved_includes
-            if x.include.endswith(_EXPLORE_FILE_EXTENSION)
+            if x.include.endswith(EXPLORE_FILE_EXTENSION)
         ]
         for included_file in explore_files:
             try:
@@ -152,9 +152,9 @@ def resolve_includes(
                 # As such, we try to handle it but are as defensive as possible.
 
                 non_base_project_name = project_name
-                if project_name == _BASE_PROJECT_NAME and root_project_name is not None:
+                if project_name == BASE_PROJECT_NAME and root_project_name is not None:
                     non_base_project_name = root_project_name
-                if non_base_project_name != _BASE_PROJECT_NAME and inc.startswith(
+                if non_base_project_name != BASE_PROJECT_NAME and inc.startswith(
                     f"/{non_base_project_name}/"
                 ):
                     # This might be a local include. Let's make sure that '/{project_name}' doesn't
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py
index f894c96debc54..9fac0b52fde0d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py
@@ -9,8 +9,8 @@
     load_and_preprocess_file,
 )
 from datahub.ingestion.source.looker.lookml_config import (
-    _EXPLORE_FILE_EXTENSION,
-    _VIEW_FILE_EXTENSION,
+    EXPLORE_FILE_EXTENSION,
+    VIEW_FILE_EXTENSION,
     LookMLSourceConfig,
     LookMLSourceReport,
 )
@@ -42,7 +42,7 @@ def _load_viewfile(
     ) -> Optional[LookerViewFile]:
         # always fully resolve paths to simplify de-dup
         path = str(pathlib.Path(path).resolve())
-        allowed_extensions = [_VIEW_FILE_EXTENSION, _EXPLORE_FILE_EXTENSION]
+        allowed_extensions = [VIEW_FILE_EXTENSION, EXPLORE_FILE_EXTENSION]
         matched_any_extension = [
             match for match in [path.endswith(x) for x in allowed_extensions] if match
         ]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
index 0b5c164a6b2c7..c3f2a110136c4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
@@ -68,6 +68,7 @@ class LookerAPIStats(BaseModel):
     get_look_calls: int = 0
     search_looks_calls: int = 0
     search_dashboards_calls: int = 0
+    all_user_calls: int = 0
 
 
 class LookerAPI:
@@ -135,7 +136,7 @@ def get_available_permissions(self) -> Set[str]:
 
         return permissions
 
-    @lru_cache(maxsize=1000)
+    @lru_cache(maxsize=5000)
     def get_user(self, id_: str, user_fields: str) -> Optional[User]:
         self.client_stats.user_calls += 1
         try:
@@ -154,6 +155,17 @@ def get_user(self, id_: str, user_fields: str) -> Optional[User]:
         # User not found
         return None
 
+    def all_users(self, user_fields: str) -> Sequence[User]:
+        self.client_stats.all_user_calls += 1
+        try:
+            return self.client.all_users(
+                fields=cast(str, user_fields),
+                transport_options=self.transport_options,
+            )
+        except SDKError as e:
+            logger.warning(f"Failure was {e}")
+        return []
+
     def execute_query(self, write_query: WriteQuery) -> List[Dict]:
         logger.debug(f"Executing query {write_query}")
         self.client_stats.query_calls += 1
@@ -196,11 +208,25 @@ def folder_ancestors(
         fields: Union[str, List[str]] = ["id", "name", "parent_id"],
     ) -> Sequence[Folder]:
         self.client_stats.folder_calls += 1
-        return self.client.folder_ancestors(
-            folder_id,
-            self.__fields_mapper(fields),
-            transport_options=self.transport_options,
-        )
+        try:
+            return self.client.folder_ancestors(
+                folder_id,
+                self.__fields_mapper(fields),
+                transport_options=self.transport_options,
+            )
+        except SDKError as e:
+            if "Looker Not Found (404)" in str(e):
+                # Folder ancestors not found
+                logger.info(
+                    f"Could not find ancestors for folder with id {folder_id}: 404 error"
+                )
+            else:
+                logger.warning(
+                    f"Could not find ancestors for folder with id {folder_id}"
+                )
+                logger.warning(f"Failure was {e}")
+        # Folder ancestors not found
+        return []
 
     def all_connections(self):
         self.client_stats.all_connections_calls += 1
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py
index 7d4ebf00cc06e..f48ba6758564b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py
@@ -4,6 +4,7 @@
 from liquid import Environment
 from liquid.ast import Node
 from liquid.context import Context
+from liquid.filter import string_filter
 from liquid.parse import expect, get_parser
 from liquid.stream import TokenStream
 from liquid.tag import Tag
@@ -81,12 +82,18 @@ def parse(self, stream: TokenStream) -> Node:
 custom_tags = [ConditionTag]
 
 
+@string_filter
+def sql_quote_filter(variable: str) -> str:
+    return f"'{variable}'"
+
+
 @lru_cache(maxsize=1)
 def _create_env() -> Environment:
-    env: Environment = Environment()
+    env: Environment = Environment(strict_filters=False)
     # register tag. One time activity
     for custom_tag in custom_tags:
         env.add_tag(custom_tag)
+    env.add_filter("sql_quote", sql_quote_filter)
     return env
 
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
index f269ccf1cd98f..8487d5113bc1d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
@@ -68,6 +68,7 @@
     ViewField,
     ViewFieldType,
     gen_model_key,
+    get_urn_looker_element_id,
 )
 from datahub.ingestion.source.looker.looker_config import LookerDashboardSourceConfig
 from datahub.ingestion.source.looker.looker_lib_wrapper import LookerAPI
@@ -139,26 +140,23 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
     """
 
     platform = "looker"
-    source_config: LookerDashboardSourceConfig
-    reporter: LookerDashboardSourceReport
-    user_registry: LookerUserRegistry
-    reachable_look_registry: Set[
-        str
-    ]  # Keep track of look-id which are reachable from Dashboard
 
     def __init__(self, config: LookerDashboardSourceConfig, ctx: PipelineContext):
         super().__init__(config, ctx)
-        self.source_config = config
-        self.reporter = LookerDashboardSourceReport()
+        self.source_config: LookerDashboardSourceConfig = config
+        self.reporter: LookerDashboardSourceReport = LookerDashboardSourceReport()
         self.looker_api: LookerAPI = LookerAPI(self.source_config)
-        self.user_registry = LookerUserRegistry(self.looker_api)
-        self.explore_registry = LookerExploreRegistry(
+        self.user_registry: LookerUserRegistry = LookerUserRegistry(
+            self.looker_api, self.reporter
+        )
+        self.explore_registry: LookerExploreRegistry = LookerExploreRegistry(
             self.looker_api, self.reporter, self.source_config
         )
         self.reporter._looker_explore_registry = self.explore_registry
         self.reporter._looker_api = self.looker_api
 
-        self.reachable_look_registry = set()
+        # Keep track of look-id which are reachable from Dashboard
+        self.reachable_look_registry: Set[str] = set()
 
         # (model, explore) -> list of charts/looks/dashboards that reference this explore
         # The list values are used purely for debugging purposes.
@@ -168,6 +166,9 @@ def __init__(self, config: LookerDashboardSourceConfig, ctx: PipelineContext):
         # Required, as we do not ingest all folders but only those that have dashboards/looks
         self.processed_folders: List[str] = []
 
+        # Keep track of ingested chart urns, to omit usage for non-ingested entities
+        self.chart_urns: Set[str] = set()
+
     @staticmethod
     def test_connection(config_dict: dict) -> TestConnectionReport:
         test_report = TestConnectionReport()
@@ -609,7 +610,6 @@ def _get_folder_browse_path_v2_entries(
     def _create_platform_instance_aspect(
         self,
     ) -> DataPlatformInstance:
-
         assert (
             self.source_config.platform_name
         ), "Platform name is not set in the configuration."
@@ -646,6 +646,7 @@ def _make_chart_metadata_events(
         chart_urn = self._make_chart_urn(
             element_id=dashboard_element.get_urn_element_id()
         )
+        self.chart_urns.add(chart_urn)
         chart_snapshot = ChartSnapshot(
             urn=chart_urn,
             aspects=[Status(removed=False)],
@@ -868,21 +869,31 @@ def _make_explore_metadata_events(
     ) -> Iterable[
         Union[MetadataChangeEvent, MetadataChangeProposalWrapper, MetadataWorkUnit]
     ]:
-        if self.source_config.emit_used_explores_only:
-            explores_to_fetch = list(self.reachable_explores.keys())
-        else:
+        if not self.source_config.emit_used_explores_only:
             explores_to_fetch = list(self.list_all_explores())
+        else:
+            # We don't keep track of project names for each explore right now.
+            # Because project names are just used for a custom property, it's
+            # fine to set them to None.
+            # TODO: Track project names for each explore.
+            explores_to_fetch = [
+                (None, model, explore)
+                for (model, explore) in self.reachable_explores.keys()
+            ]
         explores_to_fetch.sort()
 
         processed_models: List[str] = []
 
-        for model, _ in explores_to_fetch:
+        for project_name, model, _ in explores_to_fetch:
             if model not in processed_models:
                 model_key = gen_model_key(self.source_config, model)
                 yield from gen_containers(
                     container_key=model_key,
                     name=model,
                     sub_types=[BIContainerSubTypes.LOOKML_MODEL],
+                    extra_properties=(
+                        {"project": project_name} if project_name is not None else None
+                    ),
                 )
                 yield MetadataChangeProposalWrapper(
                     entityUrn=model_key.as_urn(),
@@ -896,7 +907,7 @@ def _make_explore_metadata_events(
         self.reporter.total_explores = len(explores_to_fetch)
         for future in BackpressureAwareExecutor.map(
             self.fetch_one_explore,
-            ((model, explore) for (model, explore) in explores_to_fetch),
+            ((model, explore) for (_project, model, explore) in explores_to_fetch),
             max_workers=self.source_config.max_threads,
         ):
             events, explore_id, start_time, end_time = future.result()
@@ -907,7 +918,7 @@ def _make_explore_metadata_events(
                 f"Running time of fetch_one_explore for {explore_id}: {(end_time - start_time).total_seconds()}"
             )
 
-    def list_all_explores(self) -> Iterable[Tuple[str, str]]:
+    def list_all_explores(self) -> Iterable[Tuple[Optional[str], str, str]]:
         # returns a list of (model, explore) tuples
 
         for model in self.looker_api.all_lookml_models():
@@ -916,7 +927,7 @@ def list_all_explores(self) -> Iterable[Tuple[str, str]]:
             for explore in model.explores:
                 if explore.name is None:
                     continue
-                yield (model.name, explore.name)
+                yield (model.project_name, model.name, explore.name)
 
     def fetch_one_explore(
         self, model: str, explore: str
@@ -994,7 +1005,6 @@ def _gen_folder_key(self, folder_id: str) -> LookerFolderKey:
     def _make_dashboard_and_chart_mces(
         self, looker_dashboard: LookerDashboard
     ) -> Iterable[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]]:
-
         # Step 1: Emit metadata for each Chart inside the Dashboard.
         chart_events = []
         for element in looker_dashboard.dashboard_elements:
@@ -1375,7 +1385,9 @@ def _get_folder_and_ancestors_workunits(
         yield from self._emit_folder_as_container(folder)
 
     def extract_usage_stat(
-        self, looker_dashboards: List[looker_usage.LookerDashboardForUsage]
+        self,
+        looker_dashboards: List[looker_usage.LookerDashboardForUsage],
+        ingested_chart_urns: Set[str],
     ) -> List[MetadataChangeProposalWrapper]:
         looks: List[looker_usage.LookerChartForUsage] = []
         # filter out look from all dashboard
@@ -1386,6 +1398,15 @@ def extract_usage_stat(
 
         # dedup looks
         looks = list({str(look.id): look for look in looks}.values())
+        filtered_looks = []
+        for look in looks:
+            if not look.id:
+                continue
+            chart_urn = self._make_chart_urn(get_urn_looker_element_id(look.id))
+            if chart_urn in ingested_chart_urns:
+                filtered_looks.append(look)
+            else:
+                self.reporter.charts_skipped_for_usage.add(look.id)
 
         # Keep stat generators to generate entity stat aspect later
         stat_generator_config: looker_usage.StatGeneratorConfig = (
@@ -1409,7 +1430,7 @@ def extract_usage_stat(
             stat_generator_config,
             self.reporter,
             self._make_chart_urn,
-            looks,
+            filtered_looks,
         )
 
         mcps: List[MetadataChangeProposalWrapper] = []
@@ -1664,11 +1685,20 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         if self.source_config.extract_usage_history:
             self.reporter.report_stage_start("usage_extraction")
             usage_mcps: List[MetadataChangeProposalWrapper] = self.extract_usage_stat(
-                looker_dashboards_for_usage
+                looker_dashboards_for_usage, self.chart_urns
             )
             for usage_mcp in usage_mcps:
                 yield usage_mcp.as_workunit()
             self.reporter.report_stage_end("usage_extraction")
 
+        # Dump looker user resource mappings.
+        logger.info("Ingesting looker user resource mapping workunits")
+        self.reporter.report_stage_start("user_resource_extraction")
+        yield from auto_workunit(
+            self.user_registry.to_platform_resource(
+                self.source_config.platform_instance
+            )
+        )
+
     def get_report(self) -> SourceReport:
         return self.reporter
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py
index 1e60c08fe00c2..6d49d57e07743 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py
@@ -55,7 +55,6 @@ def _create_new_liquid_variables_with_default(
             current_dict: dict = new_dict
 
             for key in keys[:-1]:
-
                 if key not in current_dict:
                     current_dict[key] = {}
 
@@ -392,7 +391,6 @@ def process_lookml_template_language(
     source_config: LookMLSourceConfig,
     view_lkml_file_dict: dict,
 ) -> None:
-
     if "views" not in view_lkml_file_dict:
         return
 
@@ -425,7 +423,6 @@ def load_and_preprocess_file(
     path: Union[str, pathlib.Path],
     source_config: LookMLSourceConfig,
 ) -> dict:
-
     parsed = load_lkml(path)
 
     process_lookml_template_language(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py
index 6a623e1e97b5d..098d7d73a3da8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py
@@ -14,7 +14,7 @@
 
 from looker_sdk.sdk.api40.models import Dashboard, LookWithQuery
 
-from datahub.emitter.mce_builder import Aspect, AspectAbstract
+from datahub.emitter.mce_builder import Aspect
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.source.looker import looker_common
 from datahub.ingestion.source.looker.looker_common import (
@@ -40,7 +40,9 @@
     DashboardUsageStatisticsClass,
     DashboardUserUsageCountsClass,
     TimeWindowSizeClass,
+    _Aspect as AspectAbstract,
 )
+from datahub.utilities.lossy_collections import LossySet
 
 logger = logging.getLogger(__name__)
 
@@ -169,7 +171,7 @@ def __init__(
         self.config = config
         self.looker_models = looker_models
         # Later it will help to find out for what are the looker entities from query result
-        self.id_vs_model: Dict[str, ModelForUsage] = {
+        self.id_to_model: Dict[str, ModelForUsage] = {
             self.get_id(looker_object): looker_object for looker_object in looker_models
         }
         self.post_filter = len(self.looker_models) > 100
@@ -224,6 +226,10 @@ def get_id(self, looker_object: ModelForUsage) -> str:
     def get_id_from_row(self, row: dict) -> str:
         pass
 
+    @abstractmethod
+    def report_skip_set(self) -> LossySet[str]:
+        pass
+
     def create_mcp(
         self, model: ModelForUsage, aspect: Aspect
     ) -> MetadataChangeProposalWrapper:
@@ -257,20 +263,11 @@ def _process_entity_timeseries_rows(
 
         return entity_stat_aspect
 
-    def _process_absolute_aspect(self) -> List[Tuple[ModelForUsage, AspectAbstract]]:
-        aspects: List[Tuple[ModelForUsage, AspectAbstract]] = []
-        for looker_object in self.looker_models:
-            aspects.append(
-                (looker_object, self.to_entity_absolute_stat_aspect(looker_object))
-            )
-
-        return aspects
-
     def _fill_user_stat_aspect(
         self,
         entity_usage_stat: Dict[Tuple[str, str], Aspect],
         user_wise_rows: List[Dict],
-    ) -> Iterable[Tuple[ModelForUsage, Aspect]]:
+    ) -> Iterable[Tuple[str, Aspect]]:
         logger.debug("Entering fill user stat aspect")
 
         # We first resolve all the users using a threadpool to warm up the cache
@@ -299,7 +296,7 @@ def _fill_user_stat_aspect(
 
         for row in user_wise_rows:
             # Confirm looker object was given for stat generation
-            looker_object = self.id_vs_model.get(self.get_id_from_row(row))
+            looker_object = self.id_to_model.get(self.get_id_from_row(row))
             if looker_object is None:
                 logger.warning(
                     "Looker object with id({}) was not register with stat generator".format(
@@ -337,7 +334,7 @@ def _fill_user_stat_aspect(
         logger.debug("Starting to yield answers for user-wise counts")
 
         for (id, _), aspect in entity_usage_stat.items():
-            yield self.id_vs_model[id], aspect
+            yield id, aspect
 
     def _execute_query(self, query: LookerQuery, query_name: str) -> List[Dict]:
         rows = []
@@ -356,7 +353,7 @@ def _execute_query(self, query: LookerQuery, query_name: str) -> List[Dict]:
             )
             if self.post_filter:
                 logger.debug("post filtering")
-                rows = [r for r in rows if self.get_id_from_row(r) in self.id_vs_model]
+                rows = [r for r in rows if self.get_id_from_row(r) in self.id_to_model]
                 logger.debug("Filtered down to %d rows", len(rows))
         except Exception as e:
             logger.warning(f"Failed to execute {query_name} query: {e}")
@@ -377,7 +374,8 @@ def generate_usage_stat_mcps(self) -> Iterable[MetadataChangeProposalWrapper]:
             return
 
         # yield absolute stat for looker entities
-        for looker_object, aspect in self._process_absolute_aspect():  # type: ignore
+        for looker_object in self.looker_models:
+            aspect = self.to_entity_absolute_stat_aspect(looker_object)
             yield self.create_mcp(looker_object, aspect)
 
         # Execute query and process the raw json which contains stat information
@@ -398,10 +396,13 @@ def generate_usage_stat_mcps(self) -> Iterable[MetadataChangeProposalWrapper]:
         )
         user_wise_rows = self._execute_query(user_wise_query_with_filters, "user_query")
         # yield absolute stat for entity
-        for looker_object, aspect in self._fill_user_stat_aspect(
+        for object_id, aspect in self._fill_user_stat_aspect(
             entity_usage_stat, user_wise_rows
         ):
-            yield self.create_mcp(looker_object, aspect)
+            if object_id in self.id_to_model:
+                yield self.create_mcp(self.id_to_model[object_id], aspect)
+            else:
+                self.report_skip_set().add(object_id)
 
 
 class DashboardStatGenerator(BaseStatGenerator):
@@ -424,6 +425,9 @@ def __init__(
     def get_stats_generator_name(self) -> str:
         return "DashboardStats"
 
+    def report_skip_set(self) -> LossySet[str]:
+        return self.report.dashboards_skipped_for_usage
+
     def get_filter(self) -> Dict[ViewField, str]:
         return {
             HistoryViewField.HISTORY_DASHBOARD_ID: ",".join(
@@ -540,6 +544,9 @@ def __init__(
     def get_stats_generator_name(self) -> str:
         return "ChartStats"
 
+    def report_skip_set(self) -> LossySet[str]:
+        return self.report.charts_skipped_for_usage
+
     def get_filter(self) -> Dict[ViewField, str]:
         return {
             LookViewField.LOOK_ID: ",".join(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py
index aa45bb72d1f46..562c7863b3134 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py
@@ -6,7 +6,7 @@
 from datahub.ingestion.source.looker.looker_dataclasses import LookerModel
 from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader
 from datahub.ingestion.source.looker.lookml_config import (
-    _BASE_PROJECT_NAME,
+    BASE_PROJECT_NAME,
     NAME,
     LookMLSourceReport,
 )
@@ -103,7 +103,7 @@ def get_looker_view_id(
 
                     current_project_name: str = (
                         include.project
-                        if include.project != _BASE_PROJECT_NAME
+                        if include.project != BASE_PROJECT_NAME
                         else self.project_name
                     )
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py
index ce4a242027e11..103f4175a9ccf 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py
@@ -88,8 +88,7 @@ def column_name_in_sql_attribute(self) -> List[str]:
         for upstream_field_match in re.finditer(r"\${TABLE}\.[\"]*([\.\w]+)", sql):
             matched_field = upstream_field_match.group(1)
             # Remove quotes from field names
-            matched_field = matched_field.replace('"', "").replace("`", "").lower()
-            column_names.append(matched_field)
+            column_names.append(matched_field.replace('"', "").replace("`", "").lower())
 
         return column_names
 
@@ -320,7 +319,6 @@ def get_including_extends(
         self,
         field: str,
     ) -> Optional[Any]:
-
         # According to Looker's inheritance rules, we need to merge the fields(i.e. dimensions, measures and
         # dimension_groups) from both the child and parent.
         if field in [DIMENSIONS, DIMENSION_GROUPS, MEASURES]:
@@ -345,7 +343,6 @@ def _get_sql_table_name_field(self) -> Optional[str]:
         return self.get_including_extends(field="sql_table_name")
 
     def _is_dot_sql_table_name_present(self) -> bool:
-
         sql_table_name: Optional[str] = self._get_sql_table_name_field()
 
         if sql_table_name is None:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py
index 0bcee14ec77a1..7ffb895349ed2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py
@@ -33,17 +33,11 @@
 
 NAME: str = "name"
 
-_BASE_PROJECT_NAME = "__BASE"
+BASE_PROJECT_NAME = "__BASE"
 
-_EXPLORE_FILE_EXTENSION = ".explore.lkml"
-
-_VIEW_FILE_EXTENSION = ".view.lkml"
-
-_MODEL_FILE_EXTENSION = ".model.lkml"
-
-VIEW_LANGUAGE_LOOKML: str = "lookml"
-
-VIEW_LANGUAGE_SQL: str = "sql"
+EXPLORE_FILE_EXTENSION = ".explore.lkml"
+VIEW_FILE_EXTENSION = ".view.lkml"
+MODEL_FILE_EXTENSION = ".model.lkml"
 
 DERIVED_VIEW_SUFFIX = r".sql_table_name"
 
@@ -124,9 +118,6 @@ class LookMLSourceConfig(
         description="List of regex patterns for LookML views to include in the extraction.",
     )
     parse_table_names_from_sql: bool = Field(True, description="See note below.")
-    sql_parser: str = Field(
-        "datahub.utilities.sql_parser.DefaultSQLParser", description="See note below."
-    )
     api: Optional[LookerAPIConfig]
     project_name: Optional[str] = Field(
         None,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py
index 892ed79754a1c..6933d9d69394b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py
@@ -5,7 +5,7 @@
 
 from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition
 from datahub.ingestion.source.looker.looker_dataclasses import LookerModel
-from datahub.ingestion.source.looker.looker_view_id_cache import LookerViewFileLoader
+from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader
 from datahub.ingestion.source.looker.lookml_config import (
     NAME,
     LookMLSourceConfig,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
index e4d8dd19fb791..c7d3724472d3c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
@@ -57,10 +57,8 @@
     LookerViewContext,
 )
 from datahub.ingestion.source.looker.lookml_config import (
-    _BASE_PROJECT_NAME,
-    _MODEL_FILE_EXTENSION,
-    VIEW_LANGUAGE_LOOKML,
-    VIEW_LANGUAGE_SQL,
+    BASE_PROJECT_NAME,
+    MODEL_FILE_EXTENSION,
     LookerConnectionDefinition,
     LookMLSourceConfig,
     LookMLSourceReport,
@@ -98,6 +96,9 @@
 )
 from datahub.sql_parsing.sqlglot_lineage import ColumnRef
 
+VIEW_LANGUAGE_LOOKML: str = "lookml"
+VIEW_LANGUAGE_SQL: str = "sql"
+
 logger = logging.getLogger(__name__)
 
 
@@ -143,7 +144,6 @@ def from_looker_dict(
         extract_col_level_lineage: bool = False,
         populate_sql_logic_in_descriptions: bool = False,
     ) -> Optional["LookerView"]:
-
         view_name = view_context.name()
 
         logger.debug(f"Handling view {view_name} in model {model_name}")
@@ -283,23 +283,21 @@ class LookMLSource(StatefulIngestionSourceBase):
     """
 
     platform = "lookml"
-    source_config: LookMLSourceConfig
-    reporter: LookMLSourceReport
-    looker_client: Optional[LookerAPI] = None
-
-    # This is populated during the git clone step.
-    base_projects_folder: Dict[str, pathlib.Path] = {}
-    remote_projects_git_info: Dict[str, GitInfo] = {}
 
     def __init__(self, config: LookMLSourceConfig, ctx: PipelineContext):
         super().__init__(config, ctx)
-        self.source_config = config
+        self.source_config: LookMLSourceConfig = config
         self.ctx = ctx
         self.reporter = LookMLSourceReport()
 
         # To keep track of projects (containers) which have already been ingested
         self.processed_projects: List[str] = []
 
+        # This is populated during the git clone step.
+        self.base_projects_folder: Dict[str, pathlib.Path] = {}
+        self.remote_projects_git_info: Dict[str, GitInfo] = {}
+
+        self.looker_client: Optional[LookerAPI] = None
         if self.source_config.api:
             self.looker_client = LookerAPI(self.source_config.api)
             self.reporter._looker_api = self.looker_client
@@ -321,7 +319,7 @@ def _load_model(self, path: str) -> LookerModel:
 
         looker_model = LookerModel.from_looker_dict(
             parsed,
-            _BASE_PROJECT_NAME,
+            BASE_PROJECT_NAME,
             self.source_config.project_name,
             self.base_projects_folder,
             path,
@@ -419,7 +417,6 @@ def _get_custom_properties(self, looker_view: LookerView) -> DatasetPropertiesCl
     def _build_dataset_mcps(
         self, looker_view: LookerView
     ) -> List[MetadataChangeProposalWrapper]:
-
         view_urn = looker_view.id.get_urn(self.source_config)
 
         subTypeEvent = MetadataChangeProposalWrapper(
@@ -503,7 +500,6 @@ def get_project_name(self, model_name: str) -> str:
     def get_manifest_if_present(self, folder: pathlib.Path) -> Optional[LookerManifest]:
         manifest_file = folder / "manifest.lkml"
         if manifest_file.exists():
-
             manifest_dict = load_and_preprocess_file(
                 path=manifest_file, source_config=self.source_config
             )
@@ -546,7 +542,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 self.source_config.base_folder = checkout_dir.resolve()
 
             self.base_projects_folder[
-                _BASE_PROJECT_NAME
+                BASE_PROJECT_NAME
             ] = self.source_config.base_folder
 
             visited_projects: Set[str] = set()
@@ -578,7 +574,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 self.base_projects_folder[project] = p_ref
 
             self._recursively_check_manifests(
-                tmp_dir, _BASE_PROJECT_NAME, visited_projects
+                tmp_dir, BASE_PROJECT_NAME, visited_projects
             )
 
             yield from self.get_internal_workunits()
@@ -609,7 +605,7 @@ def _recursively_check_manifests(
             return
 
         # Special case handling if the root project has a name in the manifest file.
-        if project_name == _BASE_PROJECT_NAME and manifest.project_name:
+        if project_name == BASE_PROJECT_NAME and manifest.project_name:
             if (
                 self.source_config.project_name is not None
                 and manifest.project_name != self.source_config.project_name
@@ -698,7 +694,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
         # The ** means "this directory and all subdirectories", and hence should
         # include all the files we want.
         model_files = sorted(
-            self.source_config.base_folder.glob(f"**/*{_MODEL_FILE_EXTENSION}")
+            self.source_config.base_folder.glob(f"**/*{MODEL_FILE_EXTENSION}")
         )
         model_suffix_len = len(".model")
 
@@ -834,7 +830,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
 
                             current_project_name: str = (
                                 include.project
-                                if include.project != _BASE_PROJECT_NAME
+                                if include.project != BASE_PROJECT_NAME
                                 else project_name
                             )
 
@@ -843,7 +839,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
                             base_folder_path: str = str(
                                 self.base_projects_folder.get(
                                     current_project_name,
-                                    self.base_projects_folder[_BASE_PROJECT_NAME],
+                                    self.base_projects_folder[BASE_PROJECT_NAME],
                                 )
                             )
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py
index 057dbca428184..971181e4300d6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py
@@ -12,6 +12,7 @@
     ViewField,
     ViewFieldType,
 )
+from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition
 from datahub.ingestion.source.looker.looker_view_id_cache import LookerViewIdCache
 from datahub.ingestion.source.looker.lookml_concept_context import (
     LookerFieldContext,
@@ -20,16 +21,17 @@
 from datahub.ingestion.source.looker.lookml_config import (
     DERIVED_VIEW_SUFFIX,
     NAME,
-    LookerConnectionDefinition,
     LookMLSourceConfig,
     LookMLSourceReport,
 )
 from datahub.ingestion.source.looker.urn_functions import get_qualified_table_name
+from datahub.sql_parsing.schema_resolver import match_columns_to_schema
 from datahub.sql_parsing.sqlglot_lineage import (
     ColumnLineageInfo,
     ColumnRef,
     SqlParsingResult,
     Urn,
+    create_and_cache_schema_resolver,
     create_lineage_sql_parsed_result,
 )
 
@@ -72,7 +74,6 @@ def resolve_derived_view_urn_of_col_ref(
     base_folder_path: str,
     config: LookMLSourceConfig,
 ) -> List[ColumnRef]:
-
     new_column_refs: List[ColumnRef] = []
     for col_ref in column_refs:
         if is_derived_view(col_ref.table.lower()):
@@ -201,7 +202,7 @@ def _generate_fully_qualified_name(
 class AbstractViewUpstream(ABC):
     """
     Implementation of this interface extracts the view upstream as per the way the view is bound to datasets.
-    For detail explanation please refer lookml_concept_context.LookerViewContext documentation.
+    For detail explanation, please refer lookml_concept_context.LookerViewContext documentation.
     """
 
     view_context: LookerViewContext
@@ -237,6 +238,47 @@ def get_upstream_dataset_urn(self) -> List[Urn]:
     def create_fields(self) -> List[ViewField]:
         return []  # it is for the special case
 
+    def create_upstream_column_refs(
+        self, upstream_urn: str, downstream_looker_columns: List[str]
+    ) -> List[ColumnRef]:
+        """
+        - **`upstream_urn`**: The URN of the upstream dataset.
+
+        - **`expected_columns`**: These are the columns identified by the Looker connector as belonging to the `upstream_urn` dataset. However, there is potential for human error in specifying the columns of the upstream dataset. For example, a user might declare a column in lowercase, while on the actual platform, it may exist in uppercase, or vice versa.
+
+        - This function ensures consistency in column-level lineage by consulting GMS before creating the final `ColumnRef` instance, avoiding discrepancies.
+        """
+        schema_resolver = create_and_cache_schema_resolver(
+            platform=self.view_context.view_connection.platform,
+            platform_instance=self.view_context.view_connection.platform_instance,
+            env=self.view_context.view_connection.platform_env or self.config.env,
+            graph=self.ctx.graph,
+        )
+
+        urn, schema_info = schema_resolver.resolve_urn(urn=upstream_urn)
+
+        if schema_info:
+            actual_columns = match_columns_to_schema(
+                schema_info, downstream_looker_columns
+            )
+        else:
+            logger.info(
+                f"schema_info not found for dataset {urn} in GMS. Using expected_columns to form ColumnRef"
+            )
+            actual_columns = [column.lower() for column in downstream_looker_columns]
+
+        upstream_column_refs: List[ColumnRef] = []
+
+        for column in actual_columns:
+            upstream_column_refs.append(
+                ColumnRef(
+                    column=column,
+                    table=upstream_urn,
+                )
+            )
+
+        return upstream_column_refs
+
 
 class SqlBasedDerivedViewUpstream(AbstractViewUpstream, ABC):
     """
@@ -373,15 +415,12 @@ def get_upstream_column_ref(
         # in-case of "select * from look_ml_view.SQL_TABLE_NAME" or extra field are defined in the looker view which is
         # referring to upstream table
         if self._get_upstream_dataset_urn() and not upstreams_column_refs:
-            upstreams_column_refs = [
-                ColumnRef(
-                    table=self._get_upstream_dataset_urn()[
-                        0
-                    ],  # 0th index has table of from clause
-                    column=column,
-                )
-                for column in field_context.column_name_in_sql_attribute()
-            ]
+            upstreams_column_refs = self.create_upstream_column_refs(
+                upstream_urn=self._get_upstream_dataset_urn()[
+                    0
+                ],  # 0th index has table of from clause,
+                downstream_looker_columns=field_context.column_name_in_sql_attribute(),
+            )
 
         # fix any derived view reference present in urn
         upstreams_column_refs = resolve_derived_view_urn_of_col_ref(
@@ -488,18 +527,18 @@ def get_upstream_column_ref(
             return upstream_column_refs
 
         explore_urn: str = self._get_upstream_dataset_urn()[0]
+        expected_columns: List[str] = []
 
         for column in field_context.column_name_in_sql_attribute():
             if column in self._get_explore_column_mapping():
                 explore_column: Dict = self._get_explore_column_mapping()[column]
-                upstream_column_refs.append(
-                    ColumnRef(
-                        column=explore_column.get("field", explore_column[NAME]),
-                        table=explore_urn,
-                    )
+                expected_columns.append(
+                    explore_column.get("field", explore_column[NAME])
                 )
 
-        return upstream_column_refs
+        return self.create_upstream_column_refs(
+            upstream_urn=explore_urn, downstream_looker_columns=expected_columns
+        )
 
     def get_upstream_dataset_urn(self) -> List[Urn]:
         return self._get_upstream_dataset_urn()
@@ -549,14 +588,10 @@ def __get_upstream_dataset_urn(self) -> Urn:
     def get_upstream_column_ref(
         self, field_context: LookerFieldContext
     ) -> List[ColumnRef]:
-        upstream_column_ref: List[ColumnRef] = []
-
-        for column_name in field_context.column_name_in_sql_attribute():
-            upstream_column_ref.append(
-                ColumnRef(table=self._get_upstream_dataset_urn(), column=column_name)
-            )
-
-        return upstream_column_ref
+        return self.create_upstream_column_refs(
+            upstream_urn=self._get_upstream_dataset_urn(),
+            downstream_looker_columns=field_context.column_name_in_sql_attribute(),
+        )
 
     def get_upstream_dataset_urn(self) -> List[Urn]:
         return [self._get_upstream_dataset_urn()]
@@ -610,15 +645,14 @@ def get_upstream_column_ref(
         self, field_context: LookerFieldContext
     ) -> List[ColumnRef]:
         upstream_column_ref: List[ColumnRef] = []
+
         if not self._get_upstream_dataset_urn():
             return upstream_column_ref
 
-        for column_name in field_context.column_name_in_sql_attribute():
-            upstream_column_ref.append(
-                ColumnRef(table=self._get_upstream_dataset_urn()[0], column=column_name)
-            )
-
-        return upstream_column_ref
+        return self.create_upstream_column_refs(
+            upstream_urn=self._get_upstream_dataset_urn()[0],
+            downstream_looker_columns=field_context.column_name_in_sql_attribute(),
+        )
 
     def get_upstream_dataset_urn(self) -> List[Urn]:
         return self._get_upstream_dataset_urn()
@@ -641,7 +675,6 @@ def create_view_upstream(
     ctx: PipelineContext,
     reporter: LookMLSourceReport,
 ) -> AbstractViewUpstream:
-
     if view_context.is_regular_case():
         return RegularViewUpstream(
             view_context=view_context,
@@ -666,7 +699,6 @@ def create_view_upstream(
             view_context.is_sql_based_derived_view_without_fields_case(),
         ]
     ):
-
         return DerivedQueryUpstreamSource(
             view_context=view_context,
             config=config,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py
index 79ec47a7efb2c..26a0331e1e576 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py
@@ -45,6 +45,9 @@ class Owners(ConfigModel):
     groups: Optional[List[str]] = None
 
 
+OwnersMultipleTypes = Union[List[Owners], Owners]
+
+
 class KnowledgeCard(ConfigModel):
     url: Optional[str] = None
     label: Optional[str] = None
@@ -57,7 +60,7 @@ class GlossaryTermConfig(ConfigModel):
     term_source: Optional[str] = None
     source_ref: Optional[str] = None
     source_url: Optional[str] = None
-    owners: Optional[Owners] = None
+    owners: Optional[OwnersMultipleTypes] = None
     inherits: Optional[List[str]] = None
     contains: Optional[List[str]] = None
     values: Optional[List[str]] = None
@@ -74,7 +77,7 @@ class GlossaryNodeConfig(ConfigModel):
     id: Optional[str] = None
     name: str
     description: str
-    owners: Optional[Owners] = None
+    owners: Optional[OwnersMultipleTypes] = None
     terms: Optional[List["GlossaryTermConfig"]] = None
     nodes: Optional[List["GlossaryNodeConfig"]] = None
     knowledge_links: Optional[List[KnowledgeCard]] = None
@@ -88,7 +91,7 @@ class DefaultConfig(ConfigModel):
     """Holds defaults for populating fields in glossary terms"""
 
     source: Optional[str] = None
-    owners: Owners
+    owners: OwnersMultipleTypes
     url: Optional[str] = None
     source_type: str = "INTERNAL"
 
@@ -153,30 +156,44 @@ def make_glossary_term_urn(
     return "urn:li:glossaryTerm:" + create_id(path, default_id, enable_auto_id)
 
 
-def get_owners(owners: Owners) -> models.OwnershipClass:
-    ownership_type, ownership_type_urn = validate_ownership_type(owners.type)
+def get_owners_multiple_types(owners: OwnersMultipleTypes) -> models.OwnershipClass:
+    """Allows owner types to be a list and maintains backward compatibility"""
+    if isinstance(owners, Owners):
+        return models.OwnershipClass(owners=list(get_owners(owners)))
+
+    owners_meta: List[models.OwnerClass] = []
+    for owner in owners:
+        owners_meta.extend(get_owners(owner))
+
+    return models.OwnershipClass(owners=owners_meta)
+
+
+def get_owners(owners: Owners) -> Iterable[models.OwnerClass]:
+    actual_type = owners.type or models.OwnershipTypeClass.DEVELOPER
+
+    if actual_type.startswith("urn:li:ownershipType:"):
+        ownership_type: str = "CUSTOM"
+        ownership_type_urn: Optional[str] = actual_type
+    else:
+        ownership_type, ownership_type_urn = validate_ownership_type(actual_type)
+
     if owners.typeUrn is not None:
         ownership_type_urn = owners.typeUrn
-    owners_meta: List[models.OwnerClass] = []
+
     if owners.users is not None:
-        owners_meta = owners_meta + [
-            models.OwnerClass(
+        for o in owners.users:
+            yield models.OwnerClass(
                 owner=make_user_urn(o),
                 type=ownership_type,
                 typeUrn=ownership_type_urn,
             )
-            for o in owners.users
-        ]
     if owners.groups is not None:
-        owners_meta = owners_meta + [
-            models.OwnerClass(
+        for o in owners.groups:
+            yield models.OwnerClass(
                 owner=make_group_urn(o),
                 type=ownership_type,
                 typeUrn=ownership_type_urn,
             )
-            for o in owners.groups
-        ]
-    return models.OwnershipClass(owners=owners_meta)
 
 
 def get_mces(
@@ -185,7 +202,7 @@ def get_mces(
     ingestion_config: BusinessGlossarySourceConfig,
     ctx: PipelineContext,
 ) -> Iterable[Union[MetadataChangeProposalWrapper, models.MetadataChangeEventClass]]:
-    root_owners = get_owners(glossary.owners)
+    root_owners = get_owners_multiple_types(glossary.owners)
 
     if glossary.nodes:
         for node in glossary.nodes:
@@ -270,7 +287,7 @@ def get_mces_from_node(
     node_owners = parentOwners
     if glossaryNode.owners is not None:
         assert glossaryNode.owners is not None
-        node_owners = get_owners(glossaryNode.owners)
+        node_owners = get_owners_multiple_types(glossaryNode.owners)
 
     node_snapshot = models.GlossaryNodeSnapshotClass(
         urn=node_urn,
@@ -426,7 +443,7 @@ def get_mces_from_term(
     ownership: models.OwnershipClass = parentOwnership
     if glossaryTerm.owners is not None:
         assert glossaryTerm.owners is not None
-        ownership = get_owners(glossaryTerm.owners)
+        ownership = get_owners_multiple_types(glossaryTerm.owners)
     aspects.append(ownership)
 
     if glossaryTerm.domain is not None:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py
index 08ed7677c7ab4..9f96f837eb9b3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py
@@ -210,7 +210,6 @@ def _get_lineage_mcp(
 
     # extract the old lineage and save it for the new mcp
     if preserve_upstream:
-
         client = get_default_graph()
 
         old_upstream_lineage = get_aspects_for_entity(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/mlflow.py b/metadata-ingestion/src/datahub/ingestion/source/mlflow.py
index cef6d2b1bb577..26d160acf330c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mlflow.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mlflow.py
@@ -38,16 +38,30 @@
 class MLflowConfig(EnvConfigMixin):
     tracking_uri: Optional[str] = Field(
         default=None,
-        description="Tracking server URI. If not set, an MLflow default tracking_uri is used (local `mlruns/` directory or `MLFLOW_TRACKING_URI` environment variable)",
+        description=(
+            "Tracking server URI. If not set, an MLflow default tracking_uri is used"
+            " (local `mlruns/` directory or `MLFLOW_TRACKING_URI` environment variable)"
+        ),
     )
     registry_uri: Optional[str] = Field(
         default=None,
-        description="Registry server URI. If not set, an MLflow default registry_uri is used (value of tracking_uri or `MLFLOW_REGISTRY_URI` environment variable)",
+        description=(
+            "Registry server URI. If not set, an MLflow default registry_uri is used"
+            " (value of tracking_uri or `MLFLOW_REGISTRY_URI` environment variable)"
+        ),
     )
     model_name_separator: str = Field(
         default="_",
         description="A string which separates model name from its version (e.g. model_1 or model-1)",
     )
+    base_external_url: Optional[str] = Field(
+        default=None,
+        description=(
+            "Base URL to use when constructing external URLs to MLflow."
+            " If not set, tracking_uri is used if it's an HTTP URL."
+            " If neither is set, external URLs are not generated."
+        ),
+    )
 
 
 @dataclass
@@ -279,12 +293,23 @@ def _make_ml_model_urn(self, model_version: ModelVersion) -> str:
         )
         return urn
 
-    def _make_external_url(self, model_version: ModelVersion) -> Union[None, str]:
+    def _get_base_external_url_from_tracking_uri(self) -> Optional[str]:
+        if isinstance(
+            self.client.tracking_uri, str
+        ) and self.client.tracking_uri.startswith("http"):
+            return self.client.tracking_uri
+        else:
+            return None
+
+    def _make_external_url(self, model_version: ModelVersion) -> Optional[str]:
         """
         Generate URL for a Model Version to MLflow UI.
         """
-        base_uri = self.client.tracking_uri
-        if base_uri.startswith("http"):
+        base_uri = (
+            self.config.base_external_url
+            or self._get_base_external_url_from_tracking_uri()
+        )
+        if base_uri:
             return f"{base_uri.rstrip('/')}/#/models/{model_version.name}/versions/{model_version.version}"
         else:
             return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py
index e24cba9b193d3..ef0b499129f97 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mode.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py
@@ -18,7 +18,6 @@
 from requests.adapters import HTTPAdapter, Retry
 from requests.exceptions import ConnectionError
 from requests.models import HTTPBasicAuth, HTTPError
-from sqllineage.runner import LineageRunner
 from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential
 
 import datahub.emitter.mce_builder as builder
@@ -99,6 +98,7 @@
     TagPropertiesClass,
     UpstreamClass,
     UpstreamLineageClass,
+    ViewPropertiesClass,
 )
 from datahub.metadata.urns import QueryUrn
 from datahub.sql_parsing.sqlglot_lineage import (
@@ -820,28 +820,6 @@ def _get_definition(self, definition_name):
             )
         return None
 
-    @lru_cache(maxsize=None)
-    def _get_source_from_query(self, raw_query: str) -> set:
-        query = self._replace_definitions(raw_query)
-        parser = LineageRunner(query)
-        source_paths = set()
-        try:
-            for table in parser.source_tables:
-                sources = str(table).split(".")
-                source_schema, source_table = sources[-2], sources[-1]
-                if source_schema == "<default>":
-                    source_schema = str(self.config.default_schema)
-
-                source_paths.add(f"{source_schema}.{source_table}")
-        except Exception as e:
-            self.report.report_failure(
-                title="Failed to Extract Lineage From Query",
-                message="Unable to retrieve lineage from Mode query.",
-                context=f"Query: {raw_query}, Error: {str(e)}",
-            )
-
-        return source_paths
-
     def _get_datasource_urn(
         self,
         platform: str,
@@ -953,16 +931,13 @@ def construct_query_or_dataset(
 
         dataset_props = DatasetPropertiesClass(
             name=report_info.get("name") if is_mode_dataset else query_data.get("name"),
-            description=f"""### Source Code
-``` sql
-{query_data.get("raw_query")}
-```
-            """,
+            description=None,
             externalUrl=externalUrl,
             customProperties=self.get_custom_props_from_dict(
                 query_data,
                 [
-                    "id" "created_at",
+                    "id",
+                    "created_at",
                     "updated_at",
                     "last_run_id",
                     "data_source_id",
@@ -972,7 +947,6 @@ def construct_query_or_dataset(
                 ],
             ),
         )
-
         yield (
             MetadataChangeProposalWrapper(
                 entityUrn=query_urn,
@@ -980,6 +954,16 @@ def construct_query_or_dataset(
             ).as_workunit()
         )
 
+        if raw_query := query_data.get("raw_query"):
+            yield MetadataChangeProposalWrapper(
+                entityUrn=query_urn,
+                aspect=ViewPropertiesClass(
+                    viewLogic=raw_query,
+                    viewLanguage=QueryLanguageClass.SQL,
+                    materialized=False,
+                ),
+            ).as_workunit()
+
         if is_mode_dataset:
             space_container_key = self.gen_space_key(space_token)
             yield from add_dataset_to_container(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
index e3fd8849ed844..bbc4897d227ba 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
@@ -50,25 +50,23 @@
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionSourceBase,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     BooleanTypeClass,
     BytesTypeClass,
+    DataPlatformInstanceClass,
+    DatasetPropertiesClass,
     NullTypeClass,
     NumberTypeClass,
     RecordTypeClass,
-    SchemaField,
-    SchemaFieldDataType,
+    SchemaFieldClass as SchemaField,
+    SchemaFieldDataTypeClass as SchemaFieldDataType,
     SchemalessClass,
-    SchemaMetadata,
+    SchemaMetadataClass as SchemaMetadata,
     StringTypeClass,
     TimeTypeClass,
     UnionTypeClass,
 )
-from datahub.metadata.schema_classes import (
-    DataPlatformInstanceClass,
-    DatasetPropertiesClass,
-)
 from datahub.metadata.urns import DatasetUrn
 
 logger = logging.getLogger(__name__)
@@ -410,6 +408,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     )
 
                 dataset_properties = DatasetPropertiesClass(
+                    name=collection_name,
                     tags=[],
                     customProperties={},
                 )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/neo4j/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/neo4j/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py b/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py
new file mode 100644
index 0000000000000..2c9107b967e4f
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py
@@ -0,0 +1,331 @@
+import logging
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Optional, Type, Union
+
+import pandas as pd
+from neo4j import GraphDatabase
+from pydantic.fields import Field
+
+from datahub.configuration.source_common import EnvConfigMixin
+from datahub.emitter.mce_builder import make_data_platform_urn, make_dataset_urn
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SupportStatus,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.common.subtypes import DatasetSubTypes
+from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaFieldDataType
+from datahub.metadata.schema_classes import (
+    AuditStampClass,
+    BooleanTypeClass,
+    DatasetPropertiesClass,
+    DateTypeClass,
+    NullTypeClass,
+    NumberTypeClass,
+    OtherSchemaClass,
+    SchemaFieldClass,
+    SchemaMetadataClass,
+    StringTypeClass,
+    SubTypesClass,
+    UnionTypeClass,
+)
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+_type_mapping: Dict[Union[Type, str], Type] = {
+    "list": UnionTypeClass,
+    "boolean": BooleanTypeClass,
+    "integer": NumberTypeClass,
+    "local_date_time": DateTypeClass,
+    "float": NumberTypeClass,
+    "string": StringTypeClass,
+    "date": DateTypeClass,
+    "node": StringTypeClass,
+    "relationship": StringTypeClass,
+}
+
+
+class Neo4jConfig(EnvConfigMixin):
+    username: str = Field(description="Neo4j Username")
+    password: str = Field(description="Neo4j Password")
+    uri: str = Field(description="The URI for the Neo4j server")
+    env: str = Field(description="Neo4j env")
+
+
+@dataclass
+class Neo4jSourceReport(SourceReport):
+    obj_failures: int = 0
+    obj_created: int = 0
+
+
+@platform_name("Neo4j", id="neo4j")
+@config_class(Neo4jConfig)
+@support_status(SupportStatus.CERTIFIED)
+class Neo4jSource(Source):
+    NODE = "node"
+    RELATIONSHIP = "relationship"
+    PLATFORM = "neo4j"
+
+    def __init__(self, ctx: PipelineContext, config: Neo4jConfig):
+        self.ctx = ctx
+        self.config = config
+        self.report = Neo4jSourceReport()
+
+    @classmethod
+    def create(cls, config_dict, ctx):
+        config = Neo4jConfig.parse_obj(config_dict)
+        return cls(ctx, config)
+
+    def get_field_type(self, attribute_type: Union[type, str]) -> SchemaFieldDataType:
+        type_class: type = _type_mapping.get(attribute_type, NullTypeClass)
+        return SchemaFieldDataType(type=type_class())
+
+    def get_schema_field_class(
+        self, col_name: str, col_type: str, **kwargs: Any
+    ) -> SchemaFieldClass:
+        if kwargs["obj_type"] == self.NODE and col_type == self.RELATIONSHIP:
+            col_type = self.NODE
+        else:
+            col_type = col_type
+        return SchemaFieldClass(
+            fieldPath=col_name,
+            type=self.get_field_type(col_type),
+            nativeDataType=col_type,
+            description=col_type.upper()
+            if col_type in (self.NODE, self.RELATIONSHIP)
+            else col_type,
+            lastModified=AuditStampClass(
+                time=round(time.time() * 1000), actor="urn:li:corpuser:ingestion"
+            ),
+        )
+
+    def add_properties(
+        self,
+        dataset: str,
+        description: Optional[str] = None,
+        custom_properties: Optional[Dict[str, str]] = None,
+    ) -> MetadataChangeProposalWrapper:
+        dataset_properties = DatasetPropertiesClass(
+            description=description,
+            customProperties=custom_properties,
+        )
+        return MetadataChangeProposalWrapper(
+            entityUrn=make_dataset_urn(
+                platform=self.PLATFORM, name=dataset, env=self.config.env
+            ),
+            aspect=dataset_properties,
+        )
+
+    def generate_neo4j_object(
+        self, dataset: str, columns: list, obj_type: Optional[str] = None
+    ) -> MetadataChangeProposalWrapper:
+        try:
+            fields = [
+                self.get_schema_field_class(key, value.lower(), obj_type=obj_type)
+                for d in columns
+                for key, value in d.items()
+            ]
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=make_dataset_urn(
+                    platform=self.PLATFORM, name=dataset, env=self.config.env
+                ),
+                aspect=SchemaMetadataClass(
+                    schemaName=dataset,
+                    platform=make_data_platform_urn(self.PLATFORM),
+                    version=0,
+                    hash="",
+                    platformSchema=OtherSchemaClass(rawSchema=""),
+                    lastModified=AuditStampClass(
+                        time=round(time.time() * 1000),
+                        actor="urn:li:corpuser:ingestion",
+                    ),
+                    fields=fields,
+                ),
+            )
+            self.report.obj_created += 1
+        except Exception as e:
+            log.error(e)
+            self.report.obj_failures += 1
+        return mcp
+
+    def get_neo4j_metadata(self, query: str) -> pd.DataFrame:
+        driver = GraphDatabase.driver(
+            self.config.uri, auth=(self.config.username, self.config.password)
+        )
+        """
+        This process retrieves the metadata for Neo4j objects using an APOC query, which returns a dictionary
+        with two columns: key and value. The key represents the Neo4j object, while the value contains the
+        corresponding metadata.
+
+        When data is returned from Neo4j, much of the relationship metadata is stored with the relevant node's
+        metadata. Consequently, the objects are organized into two separate dataframes: one for nodes and one for
+        relationships.
+
+        In the node dataframe, several fields are extracted and added as new columns. Similarly, in the relationship
+        dataframe, certain fields are parsed out, while others require metadata from the nodes dataframe.
+
+        Once the data is parsed and these two dataframes are created, we combine a subset of their columns into a
+        single dataframe, which will be used to create the DataHub objects.
+
+        See the docs for examples of metadata:  metadata-ingestion/docs/sources/neo4j/neo4j.md
+        """
+        try:
+            log.info(f"{query}")
+            with driver.session() as session:
+                result = session.run(query)
+                data = [record for record in result]
+            log.info("Closing Neo4j driver")
+            driver.close()
+
+            node_df = self.process_nodes(data)
+            rel_df = self.process_relationships(data, node_df)
+
+            union_cols = ["key", "obj_type", "property_data_types", "description"]
+            df = pd.concat([node_df[union_cols], rel_df[union_cols]])
+        except Exception as e:
+            self.report.failure(
+                message="Failed to get neo4j metadata",
+                exc=e,
+            )
+
+        return df
+
+    def process_nodes(self, data: list) -> pd.DataFrame:
+        nodes = [record for record in data if record["value"]["type"] == self.NODE]
+        node_df = pd.DataFrame(
+            nodes,
+            columns=["key", "value"],
+        )
+        node_df["obj_type"] = node_df["value"].apply(
+            lambda record: self.get_obj_type(record)
+        )
+        node_df["relationships"] = node_df["value"].apply(
+            lambda record: self.get_relationships(record)
+        )
+        node_df["properties"] = node_df["value"].apply(
+            lambda record: self.get_properties(record)
+        )
+        node_df["property_data_types"] = node_df["properties"].apply(
+            lambda record: self.get_property_data_types(record)
+        )
+        node_df["description"] = node_df.apply(
+            lambda record: self.get_node_description(record, node_df), axis=1
+        )
+        return node_df
+
+    def process_relationships(self, data: list, node_df: pd.DataFrame) -> pd.DataFrame:
+        rels = [
+            record for record in data if record["value"]["type"] == self.RELATIONSHIP
+        ]
+        rel_df = pd.DataFrame(rels, columns=["key", "value"])
+        rel_df["obj_type"] = rel_df["value"].apply(
+            lambda record: self.get_obj_type(record)
+        )
+        rel_df["properties"] = rel_df["value"].apply(
+            lambda record: self.get_properties(record)
+        )
+        rel_df["property_data_types"] = rel_df["properties"].apply(
+            lambda record: self.get_property_data_types(record)
+        )
+        rel_df["description"] = rel_df.apply(
+            lambda record: self.get_rel_descriptions(record, node_df), axis=1
+        )
+        return rel_df
+
+    def get_obj_type(self, record: dict) -> str:
+        return record["type"]
+
+    def get_rel_descriptions(self, record: dict, df: pd.DataFrame) -> str:
+        descriptions = []
+        for _, row in df.iterrows():
+            relationships = row.get("relationships", {})
+            for relationship, props in relationships.items():
+                if record["key"] == relationship:
+                    if props["direction"] == "in":
+                        for prop in props["labels"]:
+                            descriptions.append(
+                                f"({row['key']})-[{record['key']}]->({prop})"
+                            )
+        return "\n".join(descriptions)
+
+    def get_node_description(self, record: dict, df: pd.DataFrame) -> str:
+        descriptions = []
+        for _, row in df.iterrows():
+            if record["key"] == row["key"]:
+                for relationship, props in row["relationships"].items():
+                    direction = props["direction"]
+                    for node in set(props["labels"]):
+                        if direction == "in":
+                            descriptions.append(
+                                f"({row['key']})<-[{relationship}]-({node})"
+                            )
+                        elif direction == "out":
+                            descriptions.append(
+                                f"({row['key']})-[{relationship}]->({node})"
+                            )
+
+        return "\n".join(descriptions)
+
+    def get_property_data_types(self, record: dict) -> List[dict]:
+        return [{k: v["type"]} for k, v in record.items()]
+
+    def get_properties(self, record: dict) -> str:
+        return record["properties"]
+
+    def get_relationships(self, record: dict) -> dict:
+        return record.get("relationships", None)
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        df = self.get_neo4j_metadata(
+            "CALL apoc.meta.schema() YIELD value UNWIND keys(value) AS key RETURN key, value[key] AS value;"
+        )
+        for index, row in df.iterrows():
+            try:
+                yield MetadataWorkUnit(
+                    id=row["key"],
+                    mcp=self.generate_neo4j_object(
+                        columns=row["property_data_types"],
+                        dataset=row["key"],
+                    ),
+                    is_primary_source=True,
+                )
+
+                yield MetadataWorkUnit(
+                    id=row["key"],
+                    mcp=MetadataChangeProposalWrapper(
+                        entityUrn=make_dataset_urn(
+                            platform=self.PLATFORM,
+                            name=row["key"],
+                            env=self.config.env,
+                        ),
+                        aspect=SubTypesClass(
+                            typeNames=[
+                                DatasetSubTypes.NEO4J_NODE
+                                if row["obj_type"] == self.NODE
+                                else DatasetSubTypes.NEO4J_RELATIONSHIP
+                            ]
+                        ),
+                    ),
+                )
+
+                yield MetadataWorkUnit(
+                    id=row["key"],
+                    mcp=self.add_properties(
+                        dataset=row["key"],
+                        custom_properties=None,
+                        description=row["description"],
+                    ),
+                )
+
+            except Exception as e:
+                raise e
+
+    def get_report(self):
+        return self.report
diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
index 7072ebf6473df..f55d7a883edef 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
@@ -464,7 +464,6 @@ def report_dropped(self, ent_name: str) -> None:
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.LINEAGE_COARSE, "Supported. See docs for limitations")
 class NifiSource(Source):
-
     config: NifiSourceConfig
     report: NifiSourceReport
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/__init__.py
index 1068f335e8f8e..e69de29bb2d1d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/__init__.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/__init__.py
@@ -1 +0,0 @@
-from datahub.ingestion.source.powerbi.powerbi import PowerBiDashboardSource
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
index 967dd5d81112d..b49d40a0c7eb6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
@@ -1,7 +1,7 @@
 import logging
 from dataclasses import dataclass, field as dataclass_field
 from enum import Enum
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Literal, Optional, Union
 
 import pydantic
 from pydantic import validator
@@ -9,7 +9,7 @@
 
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import AllowDenyPattern, ConfigModel
-from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin
+from datahub.configuration.source_common import DatasetSourceConfigMixin, PlatformDetail
 from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.ingestion.source.common.subtypes import BIAssetSubTypes
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
@@ -19,6 +19,8 @@
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
 )
+from datahub.utilities.lossy_collections import LossyList
+from datahub.utilities.perf_timer import PerfTimer
 
 logger = logging.getLogger(__name__)
 
@@ -47,6 +49,7 @@ class Constant:
     WORKSPACE_ID = "workspaceId"
     DASHBOARD_ID = "powerbi.linkedin.com/dashboards/{}"
     DATASET_EXECUTE_QUERIES = "DATASET_EXECUTE_QUERIES_POST"
+    GET_WORKSPACE_APP = "GET_WORKSPACE_APP"
     DATASET_ID = "datasetId"
     REPORT_ID = "reportId"
     SCAN_ID = "ScanId"
@@ -118,6 +121,17 @@ class Constant:
     CHART_COUNT = "chartCount"
     WORKSPACE_NAME = "workspaceName"
     DATASET_WEB_URL = "datasetWebUrl"
+    TYPE = "type"
+    REPORT_TYPE = "reportType"
+    LAST_UPDATE = "lastUpdate"
+    APP_ID = "appId"
+    REPORTS = "reports"
+    ORIGINAL_REPORT_OBJECT_ID = "originalReportObjectId"
+    APP_SUB_TYPE = "App"
+    STATE = "state"
+    ACTIVE = "Active"
+    SQL_PARSING_FAILURE = "SQL Parsing Failure"
+    M_QUERY_NULL = '"null"'
 
 
 @dataclass
@@ -159,18 +173,41 @@ class SupportedDataPlatform(Enum):
         datahub_data_platform_name="redshift",
     )
 
-    DATABRICK_SQL = DataPlatformPair(
+    DATABRICKS_SQL = DataPlatformPair(
         powerbi_data_platform_name="Databricks", datahub_data_platform_name="databricks"
     )
 
+    DatabricksMultiCloud_SQL = DataPlatformPair(
+        powerbi_data_platform_name="DatabricksMultiCloud",
+        datahub_data_platform_name="databricks",
+    )
+
 
 @dataclass
 class PowerBiDashboardSourceReport(StaleEntityRemovalSourceReport):
+    all_workspace_count: int = 0
+    filtered_workspace_names: LossyList[str] = dataclass_field(
+        default_factory=LossyList
+    )
+    filtered_workspace_types: LossyList[str] = dataclass_field(
+        default_factory=LossyList
+    )
+
     dashboards_scanned: int = 0
     charts_scanned: int = 0
     filtered_dashboards: List[str] = dataclass_field(default_factory=list)
     filtered_charts: List[str] = dataclass_field(default_factory=list)
-    number_of_workspaces: int = 0
+
+    m_query_parse_timer: PerfTimer = dataclass_field(default_factory=PerfTimer)
+    m_query_parse_attempts: int = 0
+    m_query_parse_successes: int = 0
+    m_query_parse_timeouts: int = 0
+    m_query_parse_validation_errors: int = 0
+    m_query_parse_unexpected_character_errors: int = 0
+    m_query_parse_unknown_errors: int = 0
+    m_query_resolver_errors: int = 0
+    m_query_resolver_no_lineage: int = 0
+    m_query_resolver_successes: int = 0
 
     def report_dashboards_scanned(self, count: int = 1) -> None:
         self.dashboards_scanned += count
@@ -184,9 +221,6 @@ def report_dashboards_dropped(self, model: str) -> None:
     def report_charts_dropped(self, view: str) -> None:
         self.filtered_charts.append(view)
 
-    def report_number_of_workspaces(self, number_of_workspaces: int) -> None:
-        self.number_of_workspaces = number_of_workspaces
-
 
 def default_for_dataset_type_mapping() -> Dict[str, str]:
     dict_: dict = {}
@@ -198,19 +232,6 @@ def default_for_dataset_type_mapping() -> Dict[str, str]:
     return dict_
 
 
-class PlatformDetail(ConfigModel):
-    platform_instance: Optional[str] = pydantic.Field(
-        default=None,
-        description="DataHub platform instance name. To generate correct urn for upstream dataset, this should match "
-        "with platform instance name used in ingestion "
-        "recipe of other datahub sources.",
-    )
-    env: str = pydantic.Field(
-        default=DEFAULT_ENV,
-        description="The environment that all assets produced by DataHub platform ingestion source belong to",
-    )
-
-
 class DataBricksPlatformDetail(PlatformDetail):
     """
     metastore is an additional field used in Databricks connector to generate the dataset urn
@@ -226,7 +247,9 @@ class OwnershipMapping(ConfigModel):
         default=True, description="Whether ingest PowerBI user as Datahub Corpuser"
     )
     use_powerbi_email: bool = pydantic.Field(
-        default=False,
+        # TODO: Deprecate and remove this config, since the non-email format
+        # doesn't make much sense.
+        default=True,
         description="Use PowerBI User email to ingest as corpuser, default is powerbi user identifier",
     )
     remove_email_suffix: bool = pydantic.Field(
@@ -273,11 +296,12 @@ class PowerBiDashboardSourceConfig(
     # PowerBi workspace identifier
     workspace_id_pattern: AllowDenyPattern = pydantic.Field(
         default=AllowDenyPattern.allow_all(),
-        description="Regex patterns to filter PowerBI workspaces in ingestion",
+        description="Regex patterns to filter PowerBI workspaces in ingestion."
+        " Note: This field works in conjunction with 'workspace_type_filter' and both must be considered when filtering workspaces.",
     )
 
-    # Dataset type mapping PowerBI support many type of data-sources. Here user need to define what type of PowerBI
-    # DataSource need to be mapped to corresponding DataHub Platform DataSource. For example PowerBI `Snowflake` is
+    # Dataset type mapping PowerBI support many type of data-sources. Here user needs to define what type of PowerBI
+    # DataSource needs to be mapped to corresponding DataHub Platform DataSource. For example, PowerBI `Snowflake` is
     # mapped to DataHub `snowflake` PowerBI `PostgreSQL` is mapped to DataHub `postgres` and so on.
     dataset_type_mapping: Union[
         Dict[str, str], Dict[str, PlatformDetail]
@@ -320,8 +344,8 @@ class PowerBiDashboardSourceConfig(
     )
     workspace_id_as_urn_part: bool = pydantic.Field(
         default=False,
-        description="Highly recommend changing this to True, as you can have the same workspace name"
-        "To maintain backward compatability, this is set to False which uses workspace name",
+        description="It is recommended to set this to True only if you have legacy workspaces based on Office 365 groups, as those workspaces can have identical names. "
+        "To maintain backward compatibility, this is set to False which uses workspace name",
     )
     # Enable/Disable extracting ownership information of Dashboard
     extract_ownership: bool = pydantic.Field(
@@ -340,7 +364,7 @@ class PowerBiDashboardSourceConfig(
     )
     modified_since: Optional[str] = pydantic.Field(
         default=None,
-        description="Get only recently modified workspaces based on modified_since datetime '2023-02-10T00:00:00.0000000Z', excludePersonalWorkspaces and excludeInActiveWorkspaces limit to last 30 days",
+        description="Get only recently modified workspaces based on modified_since datetime '2023-02-10T00:00:00.0000000Z', excludeInActiveWorkspaces limit to last 30 days",
     )
     extract_dashboards: bool = pydantic.Field(
         default=True,
@@ -360,8 +384,8 @@ class PowerBiDashboardSourceConfig(
     # any existing tags defined to those entities
     extract_endorsements_to_tags: bool = pydantic.Field(
         default=False,
-        description="Whether to extract endorsements to tags, note that this may overwrite existing tags. Admin API "
-        "access is required is this setting is enabled",
+        description="Whether to extract endorsements to tags, note that this may overwrite existing tags. "
+        "Admin API access is required if this setting is enabled.",
     )
     filter_dataset_endorsements: AllowDenyPattern = pydantic.Field(
         default=AllowDenyPattern.allow_all(),
@@ -445,6 +469,40 @@ class PowerBiDashboardSourceConfig(
         description="Patch dashboard metadata",
     )
 
+    workspace_type_filter: List[
+        Literal[
+            "Workspace", "PersonalGroup", "Personal", "AdminWorkspace", "AdminInsights"
+        ]
+    ] = pydantic.Field(
+        default=["Workspace"],
+        description="Ingest the metadata of the workspace where the workspace type corresponds to the specified workspace_type_filter."
+        " Note: This field works in conjunction with 'workspace_id_pattern'. Both must be matched for a workspace to be processed.",
+    )
+
+    include_workspace_name_in_dataset_urn: bool = pydantic.Field(
+        default=False,
+        description="It is recommended to set this to true, as it helps prevent the overwriting of datasets."
+        "Read section #11560 at https://datahubproject.io/docs/how/updating-datahub/ before enabling this option."
+        "To maintain backward compatibility, this is set to False.",
+    )
+
+    extract_app: bool = pydantic.Field(
+        default=False,
+        description="Whether to ingest workspace app. Requires DataHub server 0.14.2+.",
+    )
+
+    m_query_parse_timeout: int = pydantic.Field(
+        default=70,
+        description="Timeout for PowerBI M-query parsing in seconds. Table-level lineage is determined by analyzing the M-query expression. "
+        "Increase this value if you encounter the 'M-Query Parsing Timeout' message in the connector report.",
+    )
+
+    metadata_api_timeout: int = pydantic.Field(
+        default=30,
+        description="timeout in seconds for Metadata Rest Api.",
+        hidden_from_docs=True,
+    )
+
     @root_validator(skip_on_failure=True)
     def validate_extract_column_level_lineage(cls, values: Dict) -> Dict:
         flags = [
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/dataplatform_instance_resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/dataplatform_instance_resolver.py
index baaa8d5b85ae1..6d51e853a2fb0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/dataplatform_instance_resolver.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/dataplatform_instance_resolver.py
@@ -2,8 +2,8 @@
 from abc import ABC, abstractmethod
 from typing import Union
 
+from datahub.configuration.source_common import PlatformDetail
 from datahub.ingestion.source.powerbi.config import (
-    PlatformDetail,
     PowerBiDashboardSourceConfig,
     PowerBIPlatformDetail,
 )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py
index bb0c0c2f79bbd..8c9ba3b458ab2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py
@@ -1,25 +1,18 @@
 import os
-from abc import ABC
 from dataclasses import dataclass
-from typing import Any, Dict, Optional
+from enum import Enum
+from typing import Any, Dict, List, Optional
 
 from lark import Tree
 
-TRACE_POWERBI_MQUERY_PARSER = os.getenv("DATAHUB_TRACE_POWERBI_MQUERY_PARSER", False)
-
-
-class AbstractIdentifierAccessor(ABC):  # To pass lint
-    pass
+from datahub.ingestion.source.powerbi.config import DataPlatformPair
+from datahub.sql_parsing.sqlglot_lineage import ColumnLineageInfo
 
-
-# @dataclass
-# class ItemSelector:
-#     items: Dict[str, Any]
-#     next: Optional[AbstractIdentifierAccessor]
+TRACE_POWERBI_MQUERY_PARSER = os.getenv("DATAHUB_TRACE_POWERBI_MQUERY_PARSER", False)
 
 
 @dataclass
-class IdentifierAccessor(AbstractIdentifierAccessor):
+class IdentifierAccessor:
     """
     statement
         public_order_date = Source{[Schema="public",Item="order_date"]}[Data]
@@ -30,13 +23,13 @@ class IdentifierAccessor(AbstractIdentifierAccessor):
 
         "[Schema="public",Item="order_date"]" is "items" in ItemSelector. Data of items varies as per DataSource
 
-        "public_order_date" is in "next" of ItemSelector. The "next" will be None if this identifier is leaf i.e. table
+        "public_order_date" is in "next" of ItemSelector. The "next" will be None if this identifier is leaf i.e., table
 
     """
 
     identifier: str
     items: Dict[str, Any]
-    next: Optional[AbstractIdentifierAccessor]
+    next: Optional["IdentifierAccessor"]
 
 
 @dataclass
@@ -53,3 +46,31 @@ class ReferencedTable:
     database: str
     schema: str
     table: str
+
+
+@dataclass
+class DataPlatformTable:
+    data_platform_pair: DataPlatformPair
+    urn: str
+
+
+@dataclass
+class Lineage:
+    upstreams: List[DataPlatformTable]
+    column_lineage: List[ColumnLineageInfo]
+
+    @staticmethod
+    def empty() -> "Lineage":
+        return Lineage(upstreams=[], column_lineage=[])
+
+
+class FunctionName(Enum):
+    NATIVE_QUERY = "Value.NativeQuery"
+    POSTGRESQL_DATA_ACCESS = "PostgreSQL.Database"
+    ORACLE_DATA_ACCESS = "Oracle.Database"
+    SNOWFLAKE_DATA_ACCESS = "Snowflake.Databases"
+    MSSQL_DATA_ACCESS = "Sql.Database"
+    DATABRICK_DATA_ACCESS = "Databricks.Catalogs"
+    GOOGLE_BIGQUERY_DATA_ACCESS = "GoogleBigQuery.Database"
+    AMAZON_REDSHIFT_DATA_ACCESS = "AmazonRedshift.Database"
+    DATABRICK_MULTI_CLOUD_DATA_ACCESS = "DatabricksMultiCloud.Catalogs"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py
index 27efad6dc21ca..8ffd54613eb38 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py
@@ -1,4 +1,5 @@
 import logging
+import re
 from typing import List, Optional
 
 import sqlparse
@@ -9,14 +10,29 @@
     create_lineage_sql_parsed_result,
 )
 
-SPECIAL_CHARACTERS = ["#(lf)", "(lf)", "#(tab)"]
+# It is the PowerBI M-Query way to mentioned \n , \t
+SPECIAL_CHARACTERS = {
+    "#(lf)": "\n",
+    "(lf)": "\n",
+    "#(tab)": "\t",
+}
+
+ANSI_ESCAPE_CHARACTERS = r"\x1b\[[0-9;]*m"
 
 logger = logging.getLogger(__name__)
 
 
 def remove_special_characters(native_query: str) -> str:
     for char in SPECIAL_CHARACTERS:
-        native_query = native_query.replace(char, " ")
+        native_query = native_query.replace(char, SPECIAL_CHARACTERS[char])
+
+    ansi_escape_regx = re.compile(ANSI_ESCAPE_CHARACTERS)
+
+    native_query = ansi_escape_regx.sub("", native_query)
+
+    # Replace "" quotes by ". Sqlglot is not handling column name alias surrounded with two double quotes
+
+    native_query = native_query.replace('""', '"')
 
     return native_query
 
@@ -53,6 +69,29 @@ def get_tables(native_query: str) -> List[str]:
     return tables
 
 
+def remove_drop_statement(query: str) -> str:
+    # Certain PowerBI M-Queries contain a combination of DROP and SELECT statements within SQL, causing SQLParser to fail on these queries.
+    # Therefore, these occurrences are being removed.
+
+    patterns = [
+        # Regular expression to match patterns like:
+        #   "DROP TABLE IF EXISTS #<identifier>;"
+        #   "DROP TABLE IF EXISTS #<identifier>, <identifier2>, ...;"
+        #   "DROP TABLE IF EXISTS #<identifier>, <identifier2>, ...\n"
+        r"DROP\s+TABLE\s+IF\s+EXISTS\s+(?:#?\w+(?:,\s*#?\w+)*)[;\n]",
+    ]
+
+    new_query = query
+
+    for pattern in patterns:
+        new_query = re.sub(pattern, "", new_query, flags=re.IGNORECASE)
+
+    # Remove extra spaces caused by consecutive replacements
+    new_query = re.sub(r"\s+", " ", new_query).strip()
+
+    return new_query
+
+
 def parse_custom_sql(
     ctx: PipelineContext,
     query: str,
@@ -62,15 +101,12 @@ def parse_custom_sql(
     env: str,
     platform_instance: Optional[str],
 ) -> Optional["SqlParsingResult"]:
-
     logger.debug("Using sqlglot_lineage to parse custom sql")
 
-    sql_query = remove_special_characters(query)
-
-    logger.debug(f"Processing native query = {sql_query}")
+    logger.debug(f"Processing native query using DataHub Sql Parser = {query}")
 
     return create_lineage_sql_parsed_result(
-        query=sql_query,
+        query=query,
         default_schema=schema,
         default_db=database,
         platform=platform,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py
index 3edaaed2ff814..2a5de7494920b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py
@@ -1,11 +1,13 @@
 import functools
 import importlib.resources as pkg_resource
 import logging
+import os
 from typing import Dict, List
 
 import lark
 from lark import Lark, Tree
 
+import datahub.ingestion.source.powerbi.m_query.data_classes
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.powerbi.config import (
     PowerBiDashboardSourceConfig,
@@ -19,9 +21,12 @@
     TRACE_POWERBI_MQUERY_PARSER,
 )
 from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table
+from datahub.utilities.threading_timeout import TimeoutException, threading_timeout
 
 logger = logging.getLogger(__name__)
 
+_M_QUERY_PARSE_TIMEOUT = int(os.getenv("DATAHUB_POWERBI_M_QUERY_PARSE_TIMEOUT", 60))
+
 
 @functools.lru_cache(maxsize=1)
 def get_lark_parser() -> Lark:
@@ -33,15 +38,20 @@ def get_lark_parser() -> Lark:
     return Lark(grammar, start="let_expression", regex=True)
 
 
-def _parse_expression(expression: str) -> Tree:
+def _parse_expression(expression: str, parse_timeout: int = 60) -> Tree:
     lark_parser: Lark = get_lark_parser()
 
     # Replace U+00a0 NO-BREAK SPACE with a normal space.
     # Sometimes PowerBI returns expressions with this character and it breaks the parser.
     expression = expression.replace("\u00a0", " ")
 
+    # Parser resolves the variable=null value to variable='', and in the Tree we get empty string
+    # to distinguish between an empty and null set =null to ="null"
+    expression = expression.replace("=null", '="null"')
+
     logger.debug(f"Parsing expression = {expression}")
-    parse_tree: Tree = lark_parser.parse(expression)
+    with threading_timeout(parse_timeout):
+        parse_tree: Tree = lark_parser.parse(expression)
 
     if TRACE_POWERBI_MQUERY_PARSER:
         logger.debug(parse_tree.pretty())
@@ -56,8 +66,7 @@ def get_upstream_tables(
     ctx: PipelineContext,
     config: PowerBiDashboardSourceConfig,
     parameters: Dict[str, str] = {},
-) -> List[resolver.Lineage]:
-
+) -> List[datahub.ingestion.source.powerbi.m_query.data_classes.Lineage]:
     if table.expression is None:
         logger.debug(f"There is no M-Query expression in table {table.full_name}")
         return []
@@ -69,57 +78,81 @@ def get_upstream_tables(
     )
 
     try:
-        parse_tree: Tree = _parse_expression(table.expression)
-
         valid, message = validator.validate_parse_tree(
-            parse_tree, native_query_enabled=config.native_query_parsing
+            table.expression, native_query_enabled=config.native_query_parsing
         )
         if valid is False:
             assert message is not None
             logger.debug(f"Validation failed: {message}")
             reporter.info(
-                title="Unsupported M-Query",
-                message="DataAccess function is not present in M-Query expression",
+                title="Non-Data Platform Expression",
+                message=message,
                 context=f"table-full-name={table.full_name}, expression={table.expression}, message={message}",
             )
+            reporter.m_query_parse_validation_errors += 1
             return []
+
+        with reporter.m_query_parse_timer:
+            reporter.m_query_parse_attempts += 1
+            parse_tree: Tree = _parse_expression(
+                table.expression, parse_timeout=config.m_query_parse_timeout
+            )
+
+    except KeyboardInterrupt:
+        raise
+    except TimeoutException:
+        reporter.m_query_parse_timeouts += 1
+        reporter.warning(
+            title="M-Query Parsing Timeout",
+            message=f"M-Query parsing timed out after {config.m_query_parse_timeout} seconds. Lineage for this table will not be extracted.",
+            context=f"table-full-name={table.full_name}, expression={table.expression}",
+        )
+        return []
     except (
         BaseException
     ) as e:  # TODO: Debug why BaseException is needed here and below.
         if isinstance(e, lark.exceptions.UnexpectedCharacters):
-            title = "Unexpected Character Found"
+            error_type = "Unexpected Character Error"
+            reporter.m_query_parse_unexpected_character_errors += 1
         else:
-            title = "Unknown Parsing Error"
+            error_type = "Unknown Parsing Error"
+            reporter.m_query_parse_unknown_errors += 1
 
         reporter.warning(
-            title=title,
-            message="Unknown parsing error",
+            title="Unable to parse M-Query expression",
+            message=f"Got an '{error_type}' while parsing the expression. Lineage will be missing for this table.",
             context=f"table-full-name={table.full_name}, expression={table.expression}",
             exc=e,
         )
         return []
+    reporter.m_query_parse_successes += 1
 
-    lineage: List[resolver.Lineage] = []
     try:
-        lineage = resolver.MQueryResolver(
+        lineage: List[
+            datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+        ] = resolver.MQueryResolver(
             table=table,
             parse_tree=parse_tree,
             reporter=reporter,
             parameters=parameters,
-        ).resolve_to_data_platform_table_list(
+        ).resolve_to_lineage(
             ctx=ctx,
             config=config,
             platform_instance_resolver=platform_instance_resolver,
         )
 
+        if lineage:
+            reporter.m_query_resolver_successes += 1
+        else:
+            reporter.m_query_resolver_no_lineage += 1
+        return lineage
+
     except BaseException as e:
+        reporter.m_query_resolver_errors += 1
         reporter.warning(
             title="Unknown M-Query Pattern",
             message="Encountered a unknown M-Query Expression",
             context=f"table-full-name={table.full_name}, expression={table.expression}, message={e}",
             exc=e,
         )
-
-        logger.debug(f"Stack trace for {table.full_name}:", exc_info=e)
-
-    return lineage
+        return []
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py
new file mode 100644
index 0000000000000..63520bd731de8
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py
@@ -0,0 +1,912 @@
+import logging
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, List, Optional, Tuple, Type, cast
+
+from lark import Tree
+
+from datahub.configuration.source_common import PlatformDetail
+from datahub.emitter import mce_builder as builder
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.source.powerbi.config import (
+    Constant,
+    DataBricksPlatformDetail,
+    DataPlatformPair,
+    PowerBiDashboardSourceConfig,
+    PowerBiDashboardSourceReport,
+    PowerBIPlatformDetail,
+    SupportedDataPlatform,
+)
+from datahub.ingestion.source.powerbi.dataplatform_instance_resolver import (
+    AbstractDataPlatformInstanceResolver,
+)
+from datahub.ingestion.source.powerbi.m_query import native_sql_parser, tree_function
+from datahub.ingestion.source.powerbi.m_query.data_classes import (
+    DataAccessFunctionDetail,
+    DataPlatformTable,
+    FunctionName,
+    IdentifierAccessor,
+    Lineage,
+    ReferencedTable,
+)
+from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table
+from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult
+
+logger = logging.getLogger(__name__)
+
+
+def get_next_item(items: List[str], item: str) -> Optional[str]:
+    if item in items:
+        try:
+            index = items.index(item)
+            return items[index + 1]
+        except IndexError:
+            logger.debug(f'item:"{item}", not found in item-list: {items}')
+    return None
+
+
+def urn_to_lowercase(value: str, flag: bool) -> str:
+    if flag is True:
+        return value.lower()
+
+    return value
+
+
+def make_urn(
+    config: PowerBiDashboardSourceConfig,
+    platform_instance_resolver: AbstractDataPlatformInstanceResolver,
+    data_platform_pair: DataPlatformPair,
+    server: str,
+    qualified_table_name: str,
+) -> str:
+    platform_detail: PlatformDetail = platform_instance_resolver.get_platform_instance(
+        PowerBIPlatformDetail(
+            data_platform_pair=data_platform_pair,
+            data_platform_server=server,
+        )
+    )
+
+    return builder.make_dataset_urn_with_platform_instance(
+        platform=data_platform_pair.datahub_data_platform_name,
+        platform_instance=platform_detail.platform_instance,
+        env=platform_detail.env,
+        name=urn_to_lowercase(
+            qualified_table_name, config.convert_lineage_urns_to_lowercase
+        ),
+    )
+
+
+class AbstractLineage(ABC):
+    """
+    Base class to share common functionalities among different dataplatform for M-Query parsing.
+
+    To create qualified table name we need to parse M-Query data-access-functions(https://learn.microsoft.com/en-us/powerquery-m/accessing-data-functions) and
+    the data-access-functions has some define pattern to access database-name, schema-name and table-name, for example, see below M-Query.
+
+        let
+            Source = Sql.Database("localhost", "library"),
+            dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]
+        in
+            dbo_book_issue
+
+    It is MSSQL M-Query and Sql.Database is the data-access-function to access MSSQL. If this function is available in M-Query then database name is available in the second argument of the first statement and schema-name and table-name is available in the second statement. the second statement can be repeated to access different tables from MSSQL.
+
+    DefaultTwoStepDataAccessSources extends the AbstractDataPlatformTableCreator and provides the common functionalities for data-platform which has above type of M-Query pattern
+
+    data-access-function varies as per data-platform for example for MySQL.Database for MySQL, PostgreSQL.Database for Postgres and Oracle.Database for Oracle and number of statement to
+    find out database-name , schema-name and table-name also varies as per dataplatform.
+
+    Value.NativeQuery is one of the functions which is used to execute a native query inside M-Query, for example see below M-Query
+
+        let
+            Source = Value.NativeQuery(AmazonRedshift.Database("redshift-url","dev"), "select * from dev.public.category", null, [EnableFolding=true])
+        in
+            Source
+
+    In this M-Query database-name is available in first argument and rest of the detail i.e database & schema is available in native query.
+
+    NativeQueryDataPlatformTableCreator extends AbstractDataPlatformTableCreator to support Redshift and Snowflake native query parsing.
+
+    """
+
+    ctx: PipelineContext
+    table: Table
+    config: PowerBiDashboardSourceConfig
+    reporter: PowerBiDashboardSourceReport
+    platform_instance_resolver: AbstractDataPlatformInstanceResolver
+
+    def __init__(
+        self,
+        ctx: PipelineContext,
+        table: Table,
+        config: PowerBiDashboardSourceConfig,
+        reporter: PowerBiDashboardSourceReport,
+        platform_instance_resolver: AbstractDataPlatformInstanceResolver,
+    ) -> None:
+        super().__init__()
+        self.ctx = ctx
+        self.table = table
+        self.config = config
+        self.reporter = reporter
+        self.platform_instance_resolver = platform_instance_resolver
+
+    @abstractmethod
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        pass
+
+    @abstractmethod
+    def get_platform_pair(self) -> DataPlatformPair:
+        pass
+
+    @staticmethod
+    def get_db_detail_from_argument(
+        arg_list: Tree,
+    ) -> Tuple[Optional[str], Optional[str]]:
+        arguments: List[str] = tree_function.strip_char_from_list(
+            values=tree_function.remove_whitespaces_from_list(
+                tree_function.token_values(arg_list)
+            ),
+        )
+
+        if len(arguments) < 2:
+            logger.debug(f"Expected minimum 2 arguments, but got {len(arguments)}")
+            return None, None
+
+        return arguments[0], arguments[1]
+
+    @staticmethod
+    def create_reference_table(
+        arg_list: Tree,
+        table_detail: Dict[str, str],
+    ) -> Optional[ReferencedTable]:
+        arguments: List[str] = tree_function.strip_char_from_list(
+            values=tree_function.remove_whitespaces_from_list(
+                tree_function.token_values(arg_list)
+            ),
+        )
+
+        logger.debug(f"Processing arguments {arguments}")
+
+        if (
+            len(arguments)
+            >= 4  # [0] is warehouse FQDN.
+            # [1] is endpoint, we are not using it.
+            # [2] is "Catalog" key
+            # [3] is catalog's value
+        ):
+            return ReferencedTable(
+                warehouse=arguments[0],
+                catalog=arguments[3],
+                # As per my observation, database and catalog names are same in M-Query
+                database=table_detail["Database"]
+                if table_detail.get("Database")
+                else arguments[3],
+                schema=table_detail["Schema"],
+                table=table_detail.get("Table") or table_detail["View"],
+            )
+        elif len(arguments) == 2:
+            return ReferencedTable(
+                warehouse=arguments[0],
+                database=table_detail["Database"],
+                schema=table_detail["Schema"],
+                table=table_detail.get("Table") or table_detail["View"],
+                catalog=None,
+            )
+
+        return None
+
+    def parse_custom_sql(
+        self, query: str, server: str, database: Optional[str], schema: Optional[str]
+    ) -> Lineage:
+        dataplatform_tables: List[DataPlatformTable] = []
+
+        platform_detail: PlatformDetail = (
+            self.platform_instance_resolver.get_platform_instance(
+                PowerBIPlatformDetail(
+                    data_platform_pair=self.get_platform_pair(),
+                    data_platform_server=server,
+                )
+            )
+        )
+
+        query = native_sql_parser.remove_drop_statement(
+            native_sql_parser.remove_special_characters(query)
+        )
+
+        parsed_result: Optional[
+            "SqlParsingResult"
+        ] = native_sql_parser.parse_custom_sql(
+            ctx=self.ctx,
+            query=query,
+            platform=self.get_platform_pair().datahub_data_platform_name,
+            platform_instance=platform_detail.platform_instance,
+            env=platform_detail.env,
+            database=database,
+            schema=schema,
+        )
+
+        if parsed_result is None:
+            self.reporter.info(
+                title=Constant.SQL_PARSING_FAILURE,
+                message="Fail to parse native sql present in PowerBI M-Query",
+                context=f"table-name={self.table.full_name}, sql={query}",
+            )
+            return Lineage.empty()
+
+        if parsed_result.debug_info and parsed_result.debug_info.table_error:
+            self.reporter.warning(
+                title=Constant.SQL_PARSING_FAILURE,
+                message="Fail to parse native sql present in PowerBI M-Query",
+                context=f"table-name={self.table.full_name}, error={parsed_result.debug_info.table_error},sql={query}",
+            )
+            return Lineage.empty()
+
+        for urn in parsed_result.in_tables:
+            dataplatform_tables.append(
+                DataPlatformTable(
+                    data_platform_pair=self.get_platform_pair(),
+                    urn=urn,
+                )
+            )
+
+        logger.debug(f"Native Query parsed result={parsed_result}")
+        logger.debug(f"Generated dataplatform_tables={dataplatform_tables}")
+
+        return Lineage(
+            upstreams=dataplatform_tables,
+            column_lineage=(
+                parsed_result.column_lineage
+                if parsed_result.column_lineage is not None
+                else []
+            ),
+        )
+
+
+class AmazonRedshiftLineage(AbstractLineage):
+    def get_platform_pair(self) -> DataPlatformPair:
+        return SupportedDataPlatform.AMAZON_REDSHIFT.value
+
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        logger.debug(
+            f"Processing AmazonRedshift data-access function detail {data_access_func_detail}"
+        )
+
+        server, db_name = self.get_db_detail_from_argument(
+            data_access_func_detail.arg_list
+        )
+        if db_name is None or server is None:
+            return Lineage.empty()  # Return an empty list
+
+        schema_name: str = cast(
+            IdentifierAccessor, data_access_func_detail.identifier_accessor
+        ).items["Name"]
+
+        table_name: str = cast(
+            IdentifierAccessor,
+            cast(IdentifierAccessor, data_access_func_detail.identifier_accessor).next,
+        ).items["Name"]
+
+        qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}"
+
+        urn = make_urn(
+            config=self.config,
+            platform_instance_resolver=self.platform_instance_resolver,
+            data_platform_pair=self.get_platform_pair(),
+            server=server,
+            qualified_table_name=qualified_table_name,
+        )
+
+        return Lineage(
+            upstreams=[
+                DataPlatformTable(
+                    data_platform_pair=self.get_platform_pair(),
+                    urn=urn,
+                )
+            ],
+            column_lineage=[],
+        )
+
+
+class OracleLineage(AbstractLineage):
+    def get_platform_pair(self) -> DataPlatformPair:
+        return SupportedDataPlatform.ORACLE.value
+
+    @staticmethod
+    def _get_server_and_db_name(value: str) -> Tuple[Optional[str], Optional[str]]:
+        error_message: str = (
+            f"The target argument ({value}) should in the format of <host-name>:<port>/<db-name>["
+            ".<domain>]"
+        )
+        splitter_result: List[str] = value.split("/")
+        if len(splitter_result) != 2:
+            logger.debug(error_message)
+            return None, None
+
+        db_name = splitter_result[1].split(".")[0]
+
+        return tree_function.strip_char_from_list([splitter_result[0]])[0], db_name
+
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        logger.debug(
+            f"Processing Oracle data-access function detail {data_access_func_detail}"
+        )
+
+        arguments: List[str] = tree_function.remove_whitespaces_from_list(
+            tree_function.token_values(data_access_func_detail.arg_list)
+        )
+
+        server, db_name = self._get_server_and_db_name(arguments[0])
+
+        if db_name is None or server is None:
+            return Lineage.empty()
+
+        schema_name: str = cast(
+            IdentifierAccessor, data_access_func_detail.identifier_accessor
+        ).items["Schema"]
+
+        table_name: str = cast(
+            IdentifierAccessor,
+            cast(IdentifierAccessor, data_access_func_detail.identifier_accessor).next,
+        ).items["Name"]
+
+        qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}"
+
+        urn = make_urn(
+            config=self.config,
+            platform_instance_resolver=self.platform_instance_resolver,
+            data_platform_pair=self.get_platform_pair(),
+            server=server,
+            qualified_table_name=qualified_table_name,
+        )
+
+        return Lineage(
+            upstreams=[
+                DataPlatformTable(
+                    data_platform_pair=self.get_platform_pair(),
+                    urn=urn,
+                )
+            ],
+            column_lineage=[],
+        )
+
+
+class DatabricksLineage(AbstractLineage):
+    def form_qualified_table_name(
+        self,
+        table_reference: ReferencedTable,
+        data_platform_pair: DataPlatformPair,
+    ) -> str:
+        platform_detail: PlatformDetail = (
+            self.platform_instance_resolver.get_platform_instance(
+                PowerBIPlatformDetail(
+                    data_platform_pair=data_platform_pair,
+                    data_platform_server=table_reference.warehouse,
+                )
+            )
+        )
+
+        metastore: Optional[str] = None
+
+        qualified_table_name: str = f"{table_reference.database}.{table_reference.schema}.{table_reference.table}"
+
+        if isinstance(platform_detail, DataBricksPlatformDetail):
+            metastore = platform_detail.metastore
+
+        if metastore is not None:
+            return f"{metastore}.{qualified_table_name}"
+
+        return qualified_table_name
+
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        logger.debug(
+            f"Processing Databrick data-access function detail {data_access_func_detail}"
+        )
+        table_detail: Dict[str, str] = {}
+        temp_accessor: Optional[
+            IdentifierAccessor
+        ] = data_access_func_detail.identifier_accessor
+
+        while temp_accessor:
+            # Condition to handle databricks M-query pattern where table, schema and database all are present in
+            # the same invoke statement
+            if all(
+                element in temp_accessor.items
+                for element in ["Item", "Schema", "Catalog"]
+            ):
+                table_detail["Schema"] = temp_accessor.items["Schema"]
+                table_detail["Table"] = temp_accessor.items["Item"]
+            else:
+                table_detail[temp_accessor.items["Kind"]] = temp_accessor.items["Name"]
+
+            if temp_accessor.next is not None:
+                temp_accessor = temp_accessor.next
+            else:
+                break
+
+        table_reference = self.create_reference_table(
+            arg_list=data_access_func_detail.arg_list,
+            table_detail=table_detail,
+        )
+
+        if table_reference:
+            qualified_table_name: str = self.form_qualified_table_name(
+                table_reference=table_reference,
+                data_platform_pair=self.get_platform_pair(),
+            )
+
+            urn = make_urn(
+                config=self.config,
+                platform_instance_resolver=self.platform_instance_resolver,
+                data_platform_pair=self.get_platform_pair(),
+                server=table_reference.warehouse,
+                qualified_table_name=qualified_table_name,
+            )
+
+            return Lineage(
+                upstreams=[
+                    DataPlatformTable(
+                        data_platform_pair=self.get_platform_pair(),
+                        urn=urn,
+                    )
+                ],
+                column_lineage=[],
+            )
+
+        return Lineage.empty()
+
+    def get_platform_pair(self) -> DataPlatformPair:
+        return SupportedDataPlatform.DATABRICKS_SQL.value
+
+
+class TwoStepDataAccessPattern(AbstractLineage, ABC):
+    """
+    These are the DataSource for which PowerBI Desktop generates default M-Query of the following pattern
+        let
+            Source = Sql.Database("localhost", "library"),
+            dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]
+        in
+            dbo_book_issue
+    """
+
+    def two_level_access_pattern(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        logger.debug(
+            f"Processing {self.get_platform_pair().powerbi_data_platform_name} data-access function detail {data_access_func_detail}"
+        )
+
+        server, db_name = self.get_db_detail_from_argument(
+            data_access_func_detail.arg_list
+        )
+        if server is None or db_name is None:
+            return Lineage.empty()  # Return an empty list
+
+        schema_name: str = cast(
+            IdentifierAccessor, data_access_func_detail.identifier_accessor
+        ).items["Schema"]
+
+        table_name: str = cast(
+            IdentifierAccessor, data_access_func_detail.identifier_accessor
+        ).items["Item"]
+
+        qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}"
+
+        logger.debug(
+            f"Platform({self.get_platform_pair().datahub_data_platform_name}) qualified_table_name= {qualified_table_name}"
+        )
+
+        urn = make_urn(
+            config=self.config,
+            platform_instance_resolver=self.platform_instance_resolver,
+            data_platform_pair=self.get_platform_pair(),
+            server=server,
+            qualified_table_name=qualified_table_name,
+        )
+        return Lineage(
+            upstreams=[
+                DataPlatformTable(
+                    data_platform_pair=self.get_platform_pair(),
+                    urn=urn,
+                )
+            ],
+            column_lineage=[],
+        )
+
+
+class PostgresLineage(TwoStepDataAccessPattern):
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        return self.two_level_access_pattern(data_access_func_detail)
+
+    def get_platform_pair(self) -> DataPlatformPair:
+        return SupportedDataPlatform.POSTGRES_SQL.value
+
+
+class MSSqlLineage(TwoStepDataAccessPattern):
+    # https://learn.microsoft.com/en-us/sql/relational-databases/security/authentication-access/ownership-and-user-schema-separation?view=sql-server-ver16
+    DEFAULT_SCHEMA = "dbo"  # Default schema name in MS-SQL is dbo
+
+    def get_platform_pair(self) -> DataPlatformPair:
+        return SupportedDataPlatform.MS_SQL.value
+
+    def create_urn_using_old_parser(
+        self, query: str, db_name: str, server: str
+    ) -> List[DataPlatformTable]:
+        dataplatform_tables: List[DataPlatformTable] = []
+
+        tables: List[str] = native_sql_parser.get_tables(query)
+
+        for parsed_table in tables:
+            # components: List[str] = [v.strip("[]") for v in parsed_table.split(".")]
+            components = [v.strip("[]") for v in parsed_table.split(".")]
+            if len(components) == 3:
+                database, schema, table = components
+            elif len(components) == 2:
+                schema, table = components
+                database = db_name
+            elif len(components) == 1:
+                (table,) = components
+                database = db_name
+                schema = MSSqlLineage.DEFAULT_SCHEMA
+            else:
+                self.reporter.warning(
+                    title="Invalid table format",
+                    message="The advanced SQL lineage feature (enable_advance_lineage_sql_construct) is disabled. Please either enable this feature or ensure the table is referenced as <db-name>.<schema-name>.<table-name> in the SQL.",
+                    context=f"table-name={self.table.full_name}",
+                )
+                continue
+
+            qualified_table_name = f"{database}.{schema}.{table}"
+            urn = make_urn(
+                config=self.config,
+                platform_instance_resolver=self.platform_instance_resolver,
+                data_platform_pair=self.get_platform_pair(),
+                server=server,
+                qualified_table_name=qualified_table_name,
+            )
+            dataplatform_tables.append(
+                DataPlatformTable(
+                    data_platform_pair=self.get_platform_pair(),
+                    urn=urn,
+                )
+            )
+
+        logger.debug(f"Generated upstream tables = {dataplatform_tables}")
+
+        return dataplatform_tables
+
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        arguments: List[str] = tree_function.strip_char_from_list(
+            values=tree_function.remove_whitespaces_from_list(
+                tree_function.token_values(data_access_func_detail.arg_list)
+            ),
+        )
+
+        server, database = self.get_db_detail_from_argument(
+            data_access_func_detail.arg_list
+        )
+        if server is None or database is None:
+            return Lineage.empty()  # Return an empty list
+
+        assert server
+        assert database  # to silent the lint
+
+        query: Optional[str] = get_next_item(arguments, "Query")
+        if query:
+            if self.config.enable_advance_lineage_sql_construct is False:
+                # Use previous parser to generate URN to keep backward compatibility
+                return Lineage(
+                    upstreams=self.create_urn_using_old_parser(
+                        query=query,
+                        db_name=database,
+                        server=server,
+                    ),
+                    column_lineage=[],
+                )
+
+            return self.parse_custom_sql(
+                query=query,
+                database=database,
+                server=server,
+                schema=MSSqlLineage.DEFAULT_SCHEMA,
+            )
+
+        # It is a regular case of MS-SQL
+        logger.debug("Handling with regular case")
+        return self.two_level_access_pattern(data_access_func_detail)
+
+
+class ThreeStepDataAccessPattern(AbstractLineage, ABC):
+    def get_datasource_server(
+        self, arguments: List[str], data_access_func_detail: DataAccessFunctionDetail
+    ) -> str:
+        return tree_function.strip_char_from_list([arguments[0]])[0]
+
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        logger.debug(
+            f"Processing {self.get_platform_pair().datahub_data_platform_name} function detail {data_access_func_detail}"
+        )
+
+        arguments: List[str] = tree_function.remove_whitespaces_from_list(
+            tree_function.token_values(data_access_func_detail.arg_list)
+        )
+        # First is database name
+        db_name: str = data_access_func_detail.identifier_accessor.items["Name"]  # type: ignore
+        # Second is schema name
+        schema_name: str = cast(
+            IdentifierAccessor, data_access_func_detail.identifier_accessor.next  # type: ignore
+        ).items["Name"]
+        # Third is table name
+        table_name: str = cast(
+            IdentifierAccessor, data_access_func_detail.identifier_accessor.next.next  # type: ignore
+        ).items["Name"]
+
+        qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}"
+
+        logger.debug(
+            f"{self.get_platform_pair().datahub_data_platform_name} qualified_table_name {qualified_table_name}"
+        )
+
+        server: str = self.get_datasource_server(arguments, data_access_func_detail)
+
+        urn = make_urn(
+            config=self.config,
+            platform_instance_resolver=self.platform_instance_resolver,
+            data_platform_pair=self.get_platform_pair(),
+            server=server,
+            qualified_table_name=qualified_table_name,
+        )
+
+        return Lineage(
+            upstreams=[
+                DataPlatformTable(
+                    data_platform_pair=self.get_platform_pair(),
+                    urn=urn,
+                )
+            ],
+            column_lineage=[],
+        )
+
+
+class SnowflakeLineage(ThreeStepDataAccessPattern):
+    def get_platform_pair(self) -> DataPlatformPair:
+        return SupportedDataPlatform.SNOWFLAKE.value
+
+
+class GoogleBigQueryLineage(ThreeStepDataAccessPattern):
+    def get_platform_pair(self) -> DataPlatformPair:
+        return SupportedDataPlatform.GOOGLE_BIGQUERY.value
+
+    def get_datasource_server(
+        self, arguments: List[str], data_access_func_detail: DataAccessFunctionDetail
+    ) -> str:
+        # In Google BigQuery server is project-name
+        # condition to silent lint, it is not going to be None
+        return (
+            data_access_func_detail.identifier_accessor.items["Name"]
+            if data_access_func_detail.identifier_accessor is not None
+            else ""
+        )
+
+
+class NativeQueryLineage(AbstractLineage):
+    SUPPORTED_NATIVE_QUERY_DATA_PLATFORM: dict = {
+        SupportedDataPlatform.SNOWFLAKE.value.powerbi_data_platform_name: SupportedDataPlatform.SNOWFLAKE,
+        SupportedDataPlatform.AMAZON_REDSHIFT.value.powerbi_data_platform_name: SupportedDataPlatform.AMAZON_REDSHIFT,
+        SupportedDataPlatform.DatabricksMultiCloud_SQL.value.powerbi_data_platform_name: SupportedDataPlatform.DatabricksMultiCloud_SQL,
+    }
+    current_data_platform: SupportedDataPlatform = SupportedDataPlatform.SNOWFLAKE
+
+    def get_platform_pair(self) -> DataPlatformPair:
+        return self.current_data_platform.value
+
+    @staticmethod
+    def is_native_parsing_supported(data_access_function_name: str) -> bool:
+        return (
+            data_access_function_name
+            in NativeQueryLineage.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM
+        )
+
+    def create_urn_using_old_parser(self, query: str, server: str) -> Lineage:
+        dataplatform_tables: List[DataPlatformTable] = []
+
+        tables: List[str] = native_sql_parser.get_tables(query)
+
+        for qualified_table_name in tables:
+            if len(qualified_table_name.split(".")) != 3:
+                logger.debug(
+                    f"Skipping table {qualified_table_name} as it is not as per qualified_table_name format"
+                )
+                continue
+
+            urn = make_urn(
+                config=self.config,
+                platform_instance_resolver=self.platform_instance_resolver,
+                data_platform_pair=self.get_platform_pair(),
+                server=server,
+                qualified_table_name=qualified_table_name,
+            )
+
+            dataplatform_tables.append(
+                DataPlatformTable(
+                    data_platform_pair=self.get_platform_pair(),
+                    urn=urn,
+                )
+            )
+
+        logger.debug(f"Generated dataplatform_tables {dataplatform_tables}")
+
+        return Lineage(
+            upstreams=dataplatform_tables,
+            column_lineage=[],
+        )
+
+    def get_db_name(self, data_access_tokens: List[str]) -> Optional[str]:
+        if (
+            data_access_tokens[0]
+            != SupportedDataPlatform.DatabricksMultiCloud_SQL.value.powerbi_data_platform_name
+        ):
+            return None
+
+        database: Optional[str] = get_next_item(data_access_tokens, "Database")
+
+        if (
+            database and database != Constant.M_QUERY_NULL
+        ):  # database name is explicitly set
+            return database
+
+        return get_next_item(  # database name is set in Name argument
+            data_access_tokens, "Name"
+        ) or get_next_item(  # If both above arguments are not available, then try Catalog
+            data_access_tokens, "Catalog"
+        )
+
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        t1: Optional[Tree] = tree_function.first_arg_list_func(
+            data_access_func_detail.arg_list
+        )
+        assert t1 is not None
+        flat_argument_list: List[Tree] = tree_function.flat_argument_list(t1)
+
+        if len(flat_argument_list) != 2:
+            logger.debug(
+                f"Expecting 2 argument, actual argument count is {len(flat_argument_list)}"
+            )
+            logger.debug(f"Flat argument list = {flat_argument_list}")
+            return Lineage.empty()
+
+        data_access_tokens: List[str] = tree_function.remove_whitespaces_from_list(
+            tree_function.token_values(flat_argument_list[0])
+        )
+
+        if not self.is_native_parsing_supported(data_access_tokens[0]):
+            logger.debug(
+                f"Unsupported native-query data-platform = {data_access_tokens[0]}"
+            )
+            logger.debug(
+                f"NativeQuery is supported only for {self.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM}"
+            )
+
+            return Lineage.empty()
+
+        if len(data_access_tokens[0]) < 3:
+            logger.debug(
+                f"Server is not available in argument list for data-platform {data_access_tokens[0]}. Returning empty "
+                "list"
+            )
+            return Lineage.empty()
+
+        self.current_data_platform = self.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM[
+            data_access_tokens[0]
+        ]
+        # The First argument is the query
+        sql_query: str = tree_function.strip_char_from_list(
+            values=tree_function.remove_whitespaces_from_list(
+                tree_function.token_values(flat_argument_list[1])
+            ),
+        )[
+            0
+        ]  # Remove any whitespaces and double quotes character
+
+        server = tree_function.strip_char_from_list([data_access_tokens[2]])[0]
+
+        if self.config.enable_advance_lineage_sql_construct is False:
+            # Use previous parser to generate URN to keep backward compatibility
+            return self.create_urn_using_old_parser(
+                query=sql_query,
+                server=server,
+            )
+
+        database_name: Optional[str] = self.get_db_name(data_access_tokens)
+
+        return self.parse_custom_sql(
+            query=sql_query,
+            server=server,
+            database=database_name,
+            schema=None,
+        )
+
+
+class SupportedPattern(Enum):
+    DATABRICKS_QUERY = (
+        DatabricksLineage,
+        FunctionName.DATABRICK_DATA_ACCESS,
+    )
+
+    DATABRICKS_MULTI_CLOUD = (
+        DatabricksLineage,
+        FunctionName.DATABRICK_MULTI_CLOUD_DATA_ACCESS,
+    )
+
+    POSTGRES_SQL = (
+        PostgresLineage,
+        FunctionName.POSTGRESQL_DATA_ACCESS,
+    )
+
+    ORACLE = (
+        OracleLineage,
+        FunctionName.ORACLE_DATA_ACCESS,
+    )
+
+    SNOWFLAKE = (
+        SnowflakeLineage,
+        FunctionName.SNOWFLAKE_DATA_ACCESS,
+    )
+
+    MS_SQL = (
+        MSSqlLineage,
+        FunctionName.MSSQL_DATA_ACCESS,
+    )
+
+    GOOGLE_BIG_QUERY = (
+        GoogleBigQueryLineage,
+        FunctionName.GOOGLE_BIGQUERY_DATA_ACCESS,
+    )
+
+    AMAZON_REDSHIFT = (
+        AmazonRedshiftLineage,
+        FunctionName.AMAZON_REDSHIFT_DATA_ACCESS,
+    )
+
+    NATIVE_QUERY = (
+        NativeQueryLineage,
+        FunctionName.NATIVE_QUERY,
+    )
+
+    def handler(self) -> Type[AbstractLineage]:
+        return self.value[0]
+
+    def function_name(self) -> str:
+        return self.value[1].value
+
+    @staticmethod
+    def get_function_names() -> List[str]:
+        functions: List[str] = []
+        for supported_resolver in SupportedPattern:
+            functions.append(supported_resolver.function_name())
+
+        return functions
+
+    @staticmethod
+    def get_pattern_handler(function_name: str) -> Optional["SupportedPattern"]:
+        logger.debug(f"Looking for pattern-handler for {function_name}")
+        for supported_resolver in SupportedPattern:
+            if function_name == supported_resolver.function_name():
+                return supported_resolver
+        logger.debug(f"pattern-handler not found for function_name {function_name}")
+        return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py
index 20fb0b5facbbc..2756a113d1ef0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py
@@ -1,256 +1,33 @@
 import logging
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from enum import Enum
-from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from lark import Tree
 
-import datahub.emitter.mce_builder as builder
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.powerbi.config import (
-    DataBricksPlatformDetail,
-    DataPlatformPair,
-    PlatformDetail,
     PowerBiDashboardSourceConfig,
     PowerBiDashboardSourceReport,
-    PowerBIPlatformDetail,
-    SupportedDataPlatform,
 )
 from datahub.ingestion.source.powerbi.dataplatform_instance_resolver import (
     AbstractDataPlatformInstanceResolver,
 )
-from datahub.ingestion.source.powerbi.m_query import native_sql_parser, tree_function
+from datahub.ingestion.source.powerbi.m_query import tree_function
 from datahub.ingestion.source.powerbi.m_query.data_classes import (
     TRACE_POWERBI_MQUERY_PARSER,
-    AbstractIdentifierAccessor,
     DataAccessFunctionDetail,
     IdentifierAccessor,
-    ReferencedTable,
+    Lineage,
+)
+from datahub.ingestion.source.powerbi.m_query.pattern_handler import (
+    AbstractLineage,
+    SupportedPattern,
 )
 from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table
-from datahub.sql_parsing.sqlglot_lineage import ColumnLineageInfo, SqlParsingResult
 
 logger = logging.getLogger(__name__)
 
 
-@dataclass
-class DataPlatformTable:
-    data_platform_pair: DataPlatformPair
-    urn: str
-
-
-@dataclass
-class Lineage:
-    upstreams: List[DataPlatformTable]
-    column_lineage: List[ColumnLineageInfo]
-
-    @staticmethod
-    def empty() -> "Lineage":
-        return Lineage(upstreams=[], column_lineage=[])
-
-
-def urn_to_lowercase(value: str, flag: bool) -> str:
-    if flag is True:
-        return value.lower()
-
-    return value
-
-
-def urn_creator(
-    config: PowerBiDashboardSourceConfig,
-    platform_instance_resolver: AbstractDataPlatformInstanceResolver,
-    data_platform_pair: DataPlatformPair,
-    server: str,
-    qualified_table_name: str,
-) -> str:
-
-    platform_detail: PlatformDetail = platform_instance_resolver.get_platform_instance(
-        PowerBIPlatformDetail(
-            data_platform_pair=data_platform_pair,
-            data_platform_server=server,
-        )
-    )
-
-    return builder.make_dataset_urn_with_platform_instance(
-        platform=data_platform_pair.datahub_data_platform_name,
-        platform_instance=platform_detail.platform_instance,
-        env=platform_detail.env,
-        name=urn_to_lowercase(
-            qualified_table_name, config.convert_lineage_urns_to_lowercase
-        ),
-    )
-
-
-class AbstractDataPlatformTableCreator(ABC):
-    """
-    Base class to share common functionalities among different dataplatform for M-Query parsing.
-
-    To create qualified table name we need to parse M-Query data-access-functions(https://learn.microsoft.com/en-us/powerquery-m/accessing-data-functions) and
-    the data-access-functions has some define pattern to access database-name, schema-name and table-name, for example see below M-Query.
-
-        let
-            Source = Sql.Database("localhost", "library"),
-            dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]
-        in
-            dbo_book_issue
-
-    It is MSSQL M-Query and Sql.Database is the data-access-function to access MSSQL. If this function is available in M-Query then database name is available in second argument
-    of first statement and schema-name and table-name is available in second statement. second statement can be repeated to access different tables from MSSQL.
-
-    DefaultTwoStepDataAccessSources extends the AbstractDataPlatformTableCreator and provides the common functionalities for data-platform which has above type of M-Query pattern
-
-    data-access-function varies as per data-platform for example for MySQL.Database for MySQL, PostgreSQL.Database for Postgres and Oracle.Database for Oracle and number of statement to
-    find out database-name , schema-name and table-name also varies as per dataplatform.
-
-    Value.NativeQuery is one of the function which is used to execute native query inside M-Query, for example see below M-Query
-
-        let
-            Source = Value.NativeQuery(AmazonRedshift.Database("redshift-url","dev"), "select * from dev.public.category", null, [EnableFolding=true])
-        in
-            Source
-
-    In this M-Query database-name is available in first argument and rest of the detail i.e database & schema is available in native query.
-
-    NativeQueryDataPlatformTableCreator extends AbstractDataPlatformTableCreator to support Redshift and Snowflake native query parsing.
-
-    """
-
-    ctx: PipelineContext
-    config: PowerBiDashboardSourceConfig
-    platform_instance_resolver: AbstractDataPlatformInstanceResolver
-
-    def __init__(
-        self,
-        ctx: PipelineContext,
-        config: PowerBiDashboardSourceConfig,
-        platform_instance_resolver: AbstractDataPlatformInstanceResolver,
-    ) -> None:
-        super().__init__()
-        self.ctx = ctx
-        self.config = config
-        self.platform_instance_resolver = platform_instance_resolver
-
-    @abstractmethod
-    def create_lineage(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-        pass
-
-    @abstractmethod
-    def get_platform_pair(self) -> DataPlatformPair:
-        pass
-
-    @staticmethod
-    def get_db_detail_from_argument(
-        arg_list: Tree,
-    ) -> Tuple[Optional[str], Optional[str]]:
-        arguments: List[str] = tree_function.strip_char_from_list(
-            values=tree_function.remove_whitespaces_from_list(
-                tree_function.token_values(arg_list)
-            ),
-        )
-
-        if len(arguments) < 2:
-            logger.debug(f"Expected minimum 2 arguments, but got {len(arguments)}")
-            return None, None
-
-        return arguments[0], arguments[1]
-
-    @staticmethod
-    def create_reference_table(
-        arg_list: Tree,
-        table_detail: Dict[str, str],
-    ) -> Optional[ReferencedTable]:
-
-        arguments: List[str] = tree_function.strip_char_from_list(
-            values=tree_function.remove_whitespaces_from_list(
-                tree_function.token_values(arg_list)
-            ),
-        )
-
-        logger.debug(f"Processing arguments {arguments}")
-
-        if (
-            len(arguments)
-            >= 4  # [0] is warehouse FQDN.
-            # [1] is endpoint, we are not using it.
-            # [2] is "Catalog" key
-            # [3] is catalog's value
-        ):
-            return ReferencedTable(
-                warehouse=arguments[0],
-                catalog=arguments[3],
-                # As per my observation, database and catalog names are same in M-Query
-                database=table_detail["Database"]
-                if table_detail.get("Database")
-                else arguments[3],
-                schema=table_detail["Schema"],
-                table=table_detail.get("Table") or table_detail["View"],
-            )
-        elif len(arguments) == 2:
-            return ReferencedTable(
-                warehouse=arguments[0],
-                database=table_detail["Database"],
-                schema=table_detail["Schema"],
-                table=table_detail.get("Table") or table_detail["View"],
-                catalog=None,
-            )
-
-        return None
-
-    def parse_custom_sql(
-        self, query: str, server: str, database: Optional[str], schema: Optional[str]
-    ) -> Lineage:
-
-        dataplatform_tables: List[DataPlatformTable] = []
-
-        platform_detail: PlatformDetail = (
-            self.platform_instance_resolver.get_platform_instance(
-                PowerBIPlatformDetail(
-                    data_platform_pair=self.get_platform_pair(),
-                    data_platform_server=server,
-                )
-            )
-        )
-
-        parsed_result: Optional[
-            "SqlParsingResult"
-        ] = native_sql_parser.parse_custom_sql(
-            ctx=self.ctx,
-            query=query,
-            platform=self.get_platform_pair().datahub_data_platform_name,
-            platform_instance=platform_detail.platform_instance,
-            env=platform_detail.env,
-            database=database,
-            schema=schema,
-        )
-
-        if parsed_result is None:
-            logger.debug("Failed to parse query")
-            return Lineage.empty()
-
-        for urn in parsed_result.in_tables:
-            dataplatform_tables.append(
-                DataPlatformTable(
-                    data_platform_pair=self.get_platform_pair(),
-                    urn=urn,
-                )
-            )
-
-        logger.debug(f"Native Query parsed result={parsed_result}")
-        logger.debug(f"Generated dataplatform_tables={dataplatform_tables}")
-
-        return Lineage(
-            upstreams=dataplatform_tables,
-            column_lineage=(
-                parsed_result.column_lineage
-                if parsed_result.column_lineage is not None
-                else []
-            ),
-        )
-
-
 class AbstractDataAccessMQueryResolver(ABC):
     table: Table
     parse_tree: Tree
@@ -269,10 +46,10 @@ def __init__(
         self.parse_tree = parse_tree
         self.reporter = reporter
         self.parameters = parameters
-        self.data_access_functions = SupportedResolver.get_function_names()
+        self.data_access_functions = SupportedPattern.get_function_names()
 
     @abstractmethod
-    def resolve_to_data_platform_table_list(
+    def resolve_to_lineage(
         self,
         ctx: PipelineContext,
         config: PowerBiDashboardSourceConfig,
@@ -288,10 +65,10 @@ class MQueryResolver(AbstractDataAccessMQueryResolver, ABC):
     This class has generic code to process M-Query tokens and create instance of DataAccessFunctionDetail.
 
     Once DataAccessFunctionDetail instance is initialized thereafter MQueryResolver generates the DataPlatformTable with the help of AbstractDataPlatformTableCreator
-    (see method resolve_to_data_platform_table_list).
+    (see method resolve_to_lineage).
 
-    Classes which extended from AbstractDataPlatformTableCreator knows how to convert generated DataAccessFunctionDetail instance
-    to respective DataPlatformTable instance as per dataplatform.
+    Classes which extended from AbstractDataPlatformTableCreator know how to convert generated DataAccessFunctionDetail instance
+     to the respective DataPlatformTable instance as per dataplatform.
 
     """
 
@@ -318,14 +95,12 @@ def get_item_selector_tokens(
         # remove whitespaces and quotes from token
         tokens: List[str] = tree_function.strip_char_from_list(
             tree_function.remove_whitespaces_from_list(
-                tree_function.token_values(
-                    cast(Tree, item_selector), parameters=self.parameters
-                )
+                tree_function.token_values(item_selector, parameters=self.parameters)
             ),
         )
         identifier: List[str] = tree_function.token_values(
-            cast(Tree, identifier_tree)
-        )  # type :ignore
+            identifier_tree, parameters={}
+        )
 
         # convert tokens to dict
         iterator = iter(tokens)
@@ -343,6 +118,21 @@ def get_argument_list(invoke_expression: Tree) -> Optional[Tree]:
 
         return argument_list
 
+    def take_first_argument(self, expression: Tree) -> Optional[Tree]:
+        # function is not data-access function, lets process function argument
+        first_arg_tree: Optional[Tree] = tree_function.first_arg_list_func(expression)
+
+        if first_arg_tree is None:
+            logger.debug(
+                f"Function invocation without argument in expression = {expression.pretty()}"
+            )
+            self.reporter.report_warning(
+                f"{self.table.full_name}-variable-statement",
+                "Function invocation without argument",
+            )
+            return None
+        return first_arg_tree
+
     def _process_invoke_expression(
         self, invoke_expression: Tree
     ) -> Union[DataAccessFunctionDetail, List[str], None]:
@@ -350,14 +140,18 @@ def _process_invoke_expression(
         data_access_func: str = tree_function.make_function_name(letter_tree)
         # The invoke function is either DataAccess function like PostgreSQL.Database(<argument-list>) or
         # some other function like Table.AddColumn or Table.Combine and so on
+
+        logger.debug(f"function-name: {data_access_func}")
+
         if data_access_func in self.data_access_functions:
             arg_list: Optional[Tree] = MQueryResolver.get_argument_list(
                 invoke_expression
             )
             if arg_list is None:
                 self.reporter.report_warning(
-                    f"{self.table.full_name}-arg-list",
-                    f"Argument list not found for data-access-function {data_access_func}",
+                    title="M-Query Resolver Error",
+                    message="Unable to extract lineage from parsed M-Query expression (missing argument list)",
+                    context=f"{self.table.full_name}: argument list not found for data-access-function {data_access_func}",
                 )
                 return None
 
@@ -367,19 +161,8 @@ def _process_invoke_expression(
                 identifier_accessor=None,
             )
 
-        # function is not data-access function, lets process function argument
-        first_arg_tree: Optional[Tree] = tree_function.first_arg_list_func(
-            invoke_expression
-        )
-
+        first_arg_tree: Optional[Tree] = self.take_first_argument(invoke_expression)
         if first_arg_tree is None:
-            logger.debug(
-                f"Function invocation without argument in expression = {invoke_expression.pretty()}"
-            )
-            self.reporter.report_warning(
-                f"{self.table.full_name}-variable-statement",
-                "Function invocation without argument",
-            )
             return None
 
         flat_arg_list: List[Tree] = tree_function.flat_argument_list(first_arg_tree)
@@ -388,6 +171,40 @@ def _process_invoke_expression(
             return None
 
         first_argument: Tree = flat_arg_list[0]  # take first argument only
+
+        # Detect nested function calls in the first argument
+        # M-Query's data transformation pipeline:
+        # 1. Functions typically operate on tables/columns
+        # 2. First argument must be either:
+        #    - A table variable name (referencing data source)
+        #    - Another function that eventually leads to a table
+        #
+        # Example of nested functions:
+        #   #"Split Column by Delimiter2" = Table.SplitColumn(
+        #       Table.TransformColumnTypes(#"Removed Columns1", "KB")
+        #   )
+        #
+        # In this example:
+        # - The inner function Table.TransformColumnTypes takes #"Removed Columns1"
+        #   (a table reference) as its first argument
+        # - Its result is then passed as the first argument to Table.SplitColumn
+        second_invoke_expression: Optional[
+            Tree
+        ] = tree_function.first_invoke_expression_func(first_argument)
+        if second_invoke_expression:
+            # 1. The First argument is function call
+            # 2. That function's first argument references next table variable
+            first_arg_tree = self.take_first_argument(second_invoke_expression)
+            if first_arg_tree is None:
+                return None
+
+            flat_arg_list = tree_function.flat_argument_list(first_arg_tree)
+            if len(flat_arg_list) == 0:
+                logger.debug("flat_arg_list is zero")
+                return None
+
+            first_argument = flat_arg_list[0]  # take first argument only
+
         expression: Optional[Tree] = tree_function.first_list_expression_func(
             first_argument
         )
@@ -403,8 +220,9 @@ def _process_invoke_expression(
                     f"Either list_expression or type_expression is not found = {invoke_expression.pretty()}"
                 )
                 self.reporter.report_warning(
-                    f"{self.table.full_name}-variable-statement",
-                    "Function argument expression is not supported",
+                    title="M-Query Resolver Error",
+                    message="Unable to extract lineage from parsed M-Query expression (function argument expression is not supported)",
+                    context=f"{self.table.full_name}: function argument expression is not supported",
                 )
                 return None
 
@@ -418,10 +236,10 @@ def _process_invoke_expression(
     def _process_item_selector_expression(
         self, rh_tree: Tree
     ) -> Tuple[Optional[str], Optional[Dict[str, str]]]:
-        new_identifier, key_vs_value = self.get_item_selector_tokens(  # type: ignore
-            cast(Tree, tree_function.first_expression_func(rh_tree))
-        )
+        first_expression: Optional[Tree] = tree_function.first_expression_func(rh_tree)
+        assert first_expression is not None
 
+        new_identifier, key_vs_value = self.get_item_selector_tokens(first_expression)
         return new_identifier, key_vs_value
 
     @staticmethod
@@ -473,8 +291,9 @@ def internal(
             )
             if v_statement is None:
                 self.reporter.report_warning(
-                    f"{self.table.full_name}-variable-statement",
-                    f"output variable ({current_identifier}) statement not found in table expression",
+                    title="Unable to extract lineage from M-Query expression",
+                    message="Lineage will be incomplete.",
+                    context=f"table-full-name={self.table.full_name}, expression = {self.table.expression}, output-variable={current_identifier} not found in table expression",
                 )
                 return None
 
@@ -506,7 +325,7 @@ def internal(
                 # The first argument can be a single table argument or list of table.
                 # For example Table.Combine({t1,t2},....), here first argument is list of table.
                 # Table.AddColumn(t1,....), here first argument is single table.
-                for token in cast(List[str], result):
+                for token in result:
                     internal(token, identifier_accessor)
 
             else:
@@ -528,7 +347,7 @@ def internal(
 
         return table_links
 
-    def resolve_to_data_platform_table_list(
+    def resolve_to_lineage(
         self,
         ctx: PipelineContext,
         config: PowerBiDashboardSourceConfig,
@@ -556,7 +375,7 @@ def resolve_to_data_platform_table_list(
         # Each item is data-access function
         for f_detail in table_links:
             # Get & Check if we support data-access-function available in M-Query
-            supported_resolver = SupportedResolver.get_resolver(
+            supported_resolver = SupportedPattern.get_pattern_handler(
                 f_detail.data_access_function_name
             )
             if supported_resolver is None:
@@ -569,652 +388,16 @@ def resolve_to_data_platform_table_list(
                 )
                 continue
 
-            # From supported_resolver enum get respective resolver like AmazonRedshift or Snowflake or Oracle or NativeQuery and create instance of it
-            # & also pass additional information that will be need to generate urn
-            table_qualified_name_creator: (
-                AbstractDataPlatformTableCreator
-            ) = supported_resolver.get_table_full_name_creator()(
+            # From supported_resolver enum get respective handler like AmazonRedshift or Snowflake or Oracle or NativeQuery and create instance of it
+            # & also pass additional information that will be need to generate lineage
+            pattern_handler: (AbstractLineage) = supported_resolver.handler()(
                 ctx=ctx,
+                table=self.table,
                 config=config,
+                reporter=self.reporter,
                 platform_instance_resolver=platform_instance_resolver,
             )
 
-            lineage.append(table_qualified_name_creator.create_lineage(f_detail))
+            lineage.append(pattern_handler.create_lineage(f_detail))
 
         return lineage
-
-
-class DefaultTwoStepDataAccessSources(AbstractDataPlatformTableCreator, ABC):
-    """
-    These are the DataSource for which PowerBI Desktop generates default M-Query of following pattern
-        let
-            Source = Sql.Database("localhost", "library"),
-            dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]
-        in
-            dbo_book_issue
-    """
-
-    def two_level_access_pattern(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-        logger.debug(
-            f"Processing {self.get_platform_pair().powerbi_data_platform_name} data-access function detail {data_access_func_detail}"
-        )
-
-        server, db_name = self.get_db_detail_from_argument(
-            data_access_func_detail.arg_list
-        )
-        if server is None or db_name is None:
-            return Lineage.empty()  # Return empty list
-
-        schema_name: str = cast(
-            IdentifierAccessor, data_access_func_detail.identifier_accessor
-        ).items["Schema"]
-
-        table_name: str = cast(
-            IdentifierAccessor, data_access_func_detail.identifier_accessor
-        ).items["Item"]
-
-        qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}"
-
-        logger.debug(
-            f"Platform({self.get_platform_pair().datahub_data_platform_name}) qualified_table_name= {qualified_table_name}"
-        )
-
-        urn = urn_creator(
-            config=self.config,
-            platform_instance_resolver=self.platform_instance_resolver,
-            data_platform_pair=self.get_platform_pair(),
-            server=server,
-            qualified_table_name=qualified_table_name,
-        )
-        return Lineage(
-            upstreams=[
-                DataPlatformTable(
-                    data_platform_pair=self.get_platform_pair(),
-                    urn=urn,
-                )
-            ],
-            column_lineage=[],
-        )
-
-
-class PostgresDataPlatformTableCreator(DefaultTwoStepDataAccessSources):
-    def create_lineage(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-        return self.two_level_access_pattern(data_access_func_detail)
-
-    def get_platform_pair(self) -> DataPlatformPair:
-        return SupportedDataPlatform.POSTGRES_SQL.value
-
-
-class MSSqlDataPlatformTableCreator(DefaultTwoStepDataAccessSources):
-    # https://learn.microsoft.com/en-us/sql/relational-databases/security/authentication-access/ownership-and-user-schema-separation?view=sql-server-ver16
-    DEFAULT_SCHEMA = "dbo"  # Default schema name in MS-SQL is dbo
-
-    def get_platform_pair(self) -> DataPlatformPair:
-        return SupportedDataPlatform.MS_SQL.value
-
-    def create_urn_using_old_parser(
-        self, query: str, db_name: str, server: str
-    ) -> List[DataPlatformTable]:
-        dataplatform_tables: List[DataPlatformTable] = []
-
-        tables: List[str] = native_sql_parser.get_tables(query)
-
-        for parsed_table in tables:
-            # components: List[str] = [v.strip("[]") for v in parsed_table.split(".")]
-            components = [v.strip("[]") for v in parsed_table.split(".")]
-            if len(components) == 3:
-                database, schema, table = components
-            elif len(components) == 2:
-                schema, table = components
-                database = db_name
-            elif len(components) == 1:
-                (table,) = components
-                database = db_name
-                schema = MSSqlDataPlatformTableCreator.DEFAULT_SCHEMA
-            else:
-                logger.warning(
-                    f"Unsupported table format found {parsed_table} in query {query}"
-                )
-                continue
-
-            qualified_table_name = f"{database}.{schema}.{table}"
-            urn = urn_creator(
-                config=self.config,
-                platform_instance_resolver=self.platform_instance_resolver,
-                data_platform_pair=self.get_platform_pair(),
-                server=server,
-                qualified_table_name=qualified_table_name,
-            )
-            dataplatform_tables.append(
-                DataPlatformTable(
-                    data_platform_pair=self.get_platform_pair(),
-                    urn=urn,
-                )
-            )
-
-        logger.debug(f"Generated upstream tables = {dataplatform_tables}")
-
-        return dataplatform_tables
-
-    def create_lineage(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-
-        arguments: List[str] = tree_function.strip_char_from_list(
-            values=tree_function.remove_whitespaces_from_list(
-                tree_function.token_values(data_access_func_detail.arg_list)
-            ),
-        )
-
-        if len(arguments) == 2:
-            # It is regular case of MS-SQL
-            logger.debug("Handling with regular case")
-            return self.two_level_access_pattern(data_access_func_detail)
-
-        if len(arguments) >= 4 and arguments[2] != "Query":
-            logger.debug("Unsupported case is found. Second index is not the Query")
-            return Lineage.empty()
-
-        if self.config.enable_advance_lineage_sql_construct is False:
-            # Use previous parser to generate URN to keep backward compatibility
-            return Lineage(
-                upstreams=self.create_urn_using_old_parser(
-                    query=arguments[3],
-                    db_name=arguments[1],
-                    server=arguments[0],
-                ),
-                column_lineage=[],
-            )
-
-        return self.parse_custom_sql(
-            query=arguments[3],
-            database=arguments[1],
-            server=arguments[0],
-            schema=MSSqlDataPlatformTableCreator.DEFAULT_SCHEMA,
-        )
-
-
-class OracleDataPlatformTableCreator(AbstractDataPlatformTableCreator):
-    def get_platform_pair(self) -> DataPlatformPair:
-        return SupportedDataPlatform.ORACLE.value
-
-    @staticmethod
-    def _get_server_and_db_name(value: str) -> Tuple[Optional[str], Optional[str]]:
-        error_message: str = (
-            f"The target argument ({value}) should in the format of <host-name>:<port>/<db-name>["
-            ".<domain>]"
-        )
-        splitter_result: List[str] = value.split("/")
-        if len(splitter_result) != 2:
-            logger.debug(error_message)
-            return None, None
-
-        db_name = splitter_result[1].split(".")[0]
-
-        return tree_function.strip_char_from_list([splitter_result[0]])[0], db_name
-
-    def create_lineage(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-        logger.debug(
-            f"Processing Oracle data-access function detail {data_access_func_detail}"
-        )
-
-        arguments: List[str] = tree_function.remove_whitespaces_from_list(
-            tree_function.token_values(data_access_func_detail.arg_list)
-        )
-
-        server, db_name = self._get_server_and_db_name(arguments[0])
-
-        if db_name is None or server is None:
-            return Lineage.empty()
-
-        schema_name: str = cast(
-            IdentifierAccessor, data_access_func_detail.identifier_accessor
-        ).items["Schema"]
-
-        table_name: str = cast(
-            IdentifierAccessor,
-            cast(IdentifierAccessor, data_access_func_detail.identifier_accessor).next,
-        ).items["Name"]
-
-        qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}"
-
-        urn = urn_creator(
-            config=self.config,
-            platform_instance_resolver=self.platform_instance_resolver,
-            data_platform_pair=self.get_platform_pair(),
-            server=server,
-            qualified_table_name=qualified_table_name,
-        )
-
-        return Lineage(
-            upstreams=[
-                DataPlatformTable(
-                    data_platform_pair=self.get_platform_pair(),
-                    urn=urn,
-                )
-            ],
-            column_lineage=[],
-        )
-
-
-class DatabrickDataPlatformTableCreator(AbstractDataPlatformTableCreator):
-    def form_qualified_table_name(
-        self,
-        table_reference: ReferencedTable,
-        data_platform_pair: DataPlatformPair,
-    ) -> str:
-
-        platform_detail: PlatformDetail = (
-            self.platform_instance_resolver.get_platform_instance(
-                PowerBIPlatformDetail(
-                    data_platform_pair=data_platform_pair,
-                    data_platform_server=table_reference.warehouse,
-                )
-            )
-        )
-
-        metastore: Optional[str] = None
-
-        qualified_table_name: str = f"{table_reference.database}.{table_reference.schema}.{table_reference.table}"
-
-        if isinstance(platform_detail, DataBricksPlatformDetail):
-            metastore = platform_detail.metastore
-
-        if metastore is not None:
-            return f"{metastore}.{qualified_table_name}"
-
-        return qualified_table_name
-
-    def create_lineage(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-        logger.debug(
-            f"Processing Databrick data-access function detail {data_access_func_detail}"
-        )
-        table_detail: Dict[str, str] = {}
-        temp_accessor: Optional[
-            Union[IdentifierAccessor, AbstractIdentifierAccessor]
-        ] = data_access_func_detail.identifier_accessor
-
-        while temp_accessor:
-            if isinstance(temp_accessor, IdentifierAccessor):
-                # Condition to handle databricks M-query pattern where table, schema and database all are present in
-                # the same invoke statement
-                if all(
-                    element in temp_accessor.items
-                    for element in ["Item", "Schema", "Catalog"]
-                ):
-                    table_detail["Schema"] = temp_accessor.items["Schema"]
-                    table_detail["Table"] = temp_accessor.items["Item"]
-                else:
-                    table_detail[temp_accessor.items["Kind"]] = temp_accessor.items[
-                        "Name"
-                    ]
-
-                if temp_accessor.next is not None:
-                    temp_accessor = temp_accessor.next
-                else:
-                    break
-            else:
-                logger.debug(
-                    "expecting instance to be IdentifierAccessor, please check if parsing is done properly"
-                )
-                return Lineage.empty()
-
-        table_reference = self.create_reference_table(
-            arg_list=data_access_func_detail.arg_list,
-            table_detail=table_detail,
-        )
-
-        if table_reference:
-            qualified_table_name: str = self.form_qualified_table_name(
-                table_reference=table_reference,
-                data_platform_pair=self.get_platform_pair(),
-            )
-
-            urn = urn_creator(
-                config=self.config,
-                platform_instance_resolver=self.platform_instance_resolver,
-                data_platform_pair=self.get_platform_pair(),
-                server=table_reference.warehouse,
-                qualified_table_name=qualified_table_name,
-            )
-
-            return Lineage(
-                upstreams=[
-                    DataPlatformTable(
-                        data_platform_pair=self.get_platform_pair(),
-                        urn=urn,
-                    )
-                ],
-                column_lineage=[],
-            )
-
-        return Lineage.empty()
-
-    def get_platform_pair(self) -> DataPlatformPair:
-        return SupportedDataPlatform.DATABRICK_SQL.value
-
-
-class DefaultThreeStepDataAccessSources(AbstractDataPlatformTableCreator, ABC):
-    def get_datasource_server(
-        self, arguments: List[str], data_access_func_detail: DataAccessFunctionDetail
-    ) -> str:
-        return tree_function.strip_char_from_list([arguments[0]])[0]
-
-    def create_lineage(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-        logger.debug(
-            f"Processing {self.get_platform_pair().datahub_data_platform_name} function detail {data_access_func_detail}"
-        )
-
-        arguments: List[str] = tree_function.remove_whitespaces_from_list(
-            tree_function.token_values(data_access_func_detail.arg_list)
-        )
-        # First is database name
-        db_name: str = data_access_func_detail.identifier_accessor.items["Name"]  # type: ignore
-        # Second is schema name
-        schema_name: str = cast(
-            IdentifierAccessor, data_access_func_detail.identifier_accessor.next  # type: ignore
-        ).items["Name"]
-        # Third is table name
-        table_name: str = cast(
-            IdentifierAccessor, data_access_func_detail.identifier_accessor.next.next  # type: ignore
-        ).items["Name"]
-
-        qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}"
-
-        logger.debug(
-            f"{self.get_platform_pair().datahub_data_platform_name} qualified_table_name {qualified_table_name}"
-        )
-
-        server: str = self.get_datasource_server(arguments, data_access_func_detail)
-
-        urn = urn_creator(
-            config=self.config,
-            platform_instance_resolver=self.platform_instance_resolver,
-            data_platform_pair=self.get_platform_pair(),
-            server=server,
-            qualified_table_name=qualified_table_name,
-        )
-
-        return Lineage(
-            upstreams=[
-                DataPlatformTable(
-                    data_platform_pair=self.get_platform_pair(),
-                    urn=urn,
-                )
-            ],
-            column_lineage=[],
-        )
-
-
-class SnowflakeDataPlatformTableCreator(DefaultThreeStepDataAccessSources):
-    def get_platform_pair(self) -> DataPlatformPair:
-        return SupportedDataPlatform.SNOWFLAKE.value
-
-
-class GoogleBigQueryDataPlatformTableCreator(DefaultThreeStepDataAccessSources):
-    def get_platform_pair(self) -> DataPlatformPair:
-        return SupportedDataPlatform.GOOGLE_BIGQUERY.value
-
-    def get_datasource_server(
-        self, arguments: List[str], data_access_func_detail: DataAccessFunctionDetail
-    ) -> str:
-        # In Google BigQuery server is project-name
-        # condition to silent lint, it is not going to be None
-        return (
-            data_access_func_detail.identifier_accessor.items["Name"]
-            if data_access_func_detail.identifier_accessor is not None
-            else ""
-        )
-
-
-class AmazonRedshiftDataPlatformTableCreator(AbstractDataPlatformTableCreator):
-    def get_platform_pair(self) -> DataPlatformPair:
-        return SupportedDataPlatform.AMAZON_REDSHIFT.value
-
-    def create_lineage(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-        logger.debug(
-            f"Processing AmazonRedshift data-access function detail {data_access_func_detail}"
-        )
-
-        server, db_name = self.get_db_detail_from_argument(
-            data_access_func_detail.arg_list
-        )
-        if db_name is None or server is None:
-            return Lineage.empty()  # Return empty list
-
-        schema_name: str = cast(
-            IdentifierAccessor, data_access_func_detail.identifier_accessor
-        ).items["Name"]
-
-        table_name: str = cast(
-            IdentifierAccessor,
-            cast(IdentifierAccessor, data_access_func_detail.identifier_accessor).next,
-        ).items["Name"]
-
-        qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}"
-
-        urn = urn_creator(
-            config=self.config,
-            platform_instance_resolver=self.platform_instance_resolver,
-            data_platform_pair=self.get_platform_pair(),
-            server=server,
-            qualified_table_name=qualified_table_name,
-        )
-
-        return Lineage(
-            upstreams=[
-                DataPlatformTable(
-                    data_platform_pair=self.get_platform_pair(),
-                    urn=urn,
-                )
-            ],
-            column_lineage=[],
-        )
-
-
-class NativeQueryDataPlatformTableCreator(AbstractDataPlatformTableCreator):
-    SUPPORTED_NATIVE_QUERY_DATA_PLATFORM: dict = {
-        SupportedDataPlatform.SNOWFLAKE.value.powerbi_data_platform_name: SupportedDataPlatform.SNOWFLAKE,
-        SupportedDataPlatform.AMAZON_REDSHIFT.value.powerbi_data_platform_name: SupportedDataPlatform.AMAZON_REDSHIFT,
-    }
-    current_data_platform: SupportedDataPlatform = SupportedDataPlatform.SNOWFLAKE
-
-    def get_platform_pair(self) -> DataPlatformPair:
-        return self.current_data_platform.value
-
-    @staticmethod
-    def is_native_parsing_supported(data_access_function_name: str) -> bool:
-        return (
-            data_access_function_name
-            in NativeQueryDataPlatformTableCreator.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM
-        )
-
-    def create_urn_using_old_parser(self, query: str, server: str) -> Lineage:
-        dataplatform_tables: List[DataPlatformTable] = []
-
-        tables: List[str] = native_sql_parser.get_tables(query)
-
-        for qualified_table_name in tables:
-            if len(qualified_table_name.split(".")) != 3:
-                logger.debug(
-                    f"Skipping table {qualified_table_name} as it is not as per qualified_table_name format"
-                )
-                continue
-
-            urn = urn_creator(
-                config=self.config,
-                platform_instance_resolver=self.platform_instance_resolver,
-                data_platform_pair=self.get_platform_pair(),
-                server=server,
-                qualified_table_name=qualified_table_name,
-            )
-
-            dataplatform_tables.append(
-                DataPlatformTable(
-                    data_platform_pair=self.get_platform_pair(),
-                    urn=urn,
-                )
-            )
-
-        logger.debug(f"Generated dataplatform_tables {dataplatform_tables}")
-
-        return Lineage(
-            upstreams=dataplatform_tables,
-            column_lineage=[],
-        )
-
-    def create_lineage(
-        self, data_access_func_detail: DataAccessFunctionDetail
-    ) -> Lineage:
-        t1: Tree = cast(
-            Tree, tree_function.first_arg_list_func(data_access_func_detail.arg_list)
-        )
-        flat_argument_list: List[Tree] = tree_function.flat_argument_list(t1)
-
-        if len(flat_argument_list) != 2:
-            logger.debug(
-                f"Expecting 2 argument, actual argument count is {len(flat_argument_list)}"
-            )
-            logger.debug(f"Flat argument list = {flat_argument_list}")
-            return Lineage.empty()
-        data_access_tokens: List[str] = tree_function.remove_whitespaces_from_list(
-            tree_function.token_values(flat_argument_list[0])
-        )
-
-        if not self.is_native_parsing_supported(data_access_tokens[0]):
-            logger.debug(
-                f"Unsupported native-query data-platform = {data_access_tokens[0]}"
-            )
-            logger.debug(
-                f"NativeQuery is supported only for {self.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM}"
-            )
-
-        if len(data_access_tokens[0]) < 3:
-            logger.debug(
-                f"Server is not available in argument list for data-platform {data_access_tokens[0]}. Returning empty "
-                "list"
-            )
-            return Lineage.empty()
-
-        self.current_data_platform = self.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM[
-            data_access_tokens[0]
-        ]
-
-        # First argument is the query
-        sql_query: str = tree_function.strip_char_from_list(
-            values=tree_function.remove_whitespaces_from_list(
-                tree_function.token_values(flat_argument_list[1])
-            ),
-        )[
-            0
-        ]  # Remove any whitespaces and double quotes character
-
-        server = tree_function.strip_char_from_list([data_access_tokens[2]])[0]
-
-        if self.config.enable_advance_lineage_sql_construct is False:
-            # Use previous parser to generate URN to keep backward compatibility
-            return self.create_urn_using_old_parser(
-                query=sql_query,
-                server=server,
-            )
-
-        return self.parse_custom_sql(
-            query=sql_query,
-            server=server,
-            database=None,  # database and schema is available inside custom sql as per PowerBI Behavior
-            schema=None,
-        )
-
-
-class FunctionName(Enum):
-    NATIVE_QUERY = "Value.NativeQuery"
-    POSTGRESQL_DATA_ACCESS = "PostgreSQL.Database"
-    ORACLE_DATA_ACCESS = "Oracle.Database"
-    SNOWFLAKE_DATA_ACCESS = "Snowflake.Databases"
-    MSSQL_DATA_ACCESS = "Sql.Database"
-    DATABRICK_DATA_ACCESS = "Databricks.Catalogs"
-    GOOGLE_BIGQUERY_DATA_ACCESS = "GoogleBigQuery.Database"
-    AMAZON_REDSHIFT_DATA_ACCESS = "AmazonRedshift.Database"
-    DATABRICK_MULTI_CLOUD_DATA_ACCESS = "DatabricksMultiCloud.Catalogs"
-
-
-class SupportedResolver(Enum):
-    DATABRICKS_QUERY = (
-        DatabrickDataPlatformTableCreator,
-        FunctionName.DATABRICK_DATA_ACCESS,
-    )
-
-    DATABRICKS_MULTI_CLOUD = (
-        DatabrickDataPlatformTableCreator,
-        FunctionName.DATABRICK_MULTI_CLOUD_DATA_ACCESS,
-    )
-
-    POSTGRES_SQL = (
-        PostgresDataPlatformTableCreator,
-        FunctionName.POSTGRESQL_DATA_ACCESS,
-    )
-
-    ORACLE = (
-        OracleDataPlatformTableCreator,
-        FunctionName.ORACLE_DATA_ACCESS,
-    )
-
-    SNOWFLAKE = (
-        SnowflakeDataPlatformTableCreator,
-        FunctionName.SNOWFLAKE_DATA_ACCESS,
-    )
-
-    MS_SQL = (
-        MSSqlDataPlatformTableCreator,
-        FunctionName.MSSQL_DATA_ACCESS,
-    )
-
-    GOOGLE_BIG_QUERY = (
-        GoogleBigQueryDataPlatformTableCreator,
-        FunctionName.GOOGLE_BIGQUERY_DATA_ACCESS,
-    )
-
-    AMAZON_REDSHIFT = (
-        AmazonRedshiftDataPlatformTableCreator,
-        FunctionName.AMAZON_REDSHIFT_DATA_ACCESS,
-    )
-
-    NATIVE_QUERY = (
-        NativeQueryDataPlatformTableCreator,
-        FunctionName.NATIVE_QUERY,
-    )
-
-    def get_table_full_name_creator(self) -> Type[AbstractDataPlatformTableCreator]:
-        return self.value[0]
-
-    def get_function_name(self) -> str:
-        return self.value[1].value
-
-    @staticmethod
-    def get_function_names() -> List[str]:
-        functions: List[str] = []
-        for supported_resolver in SupportedResolver:
-            functions.append(supported_resolver.get_function_name())
-
-        return functions
-
-    @staticmethod
-    def get_resolver(function_name: str) -> Optional["SupportedResolver"]:
-        logger.debug(f"Looking for resolver {function_name}")
-        for supported_resolver in SupportedResolver:
-            if function_name == supported_resolver.get_function_name():
-                return supported_resolver
-        logger.debug(f"Resolver not found for function_name {function_name}")
-        return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/tree_function.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/tree_function.py
index dfcbcf1e6f462..d48e251bd0090 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/tree_function.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/tree_function.py
@@ -1,6 +1,6 @@
 import logging
 from functools import partial
-from typing import Any, Dict, List, Optional, Union, cast
+from typing import Any, Dict, List, Optional, Union
 
 from lark import Token, Tree
 
@@ -46,8 +46,8 @@ def get_variable_statement(parse_tree: Tree, variable: str) -> Optional[Tree]:
 
 def get_first_rule(tree: Tree, rule: str) -> Optional[Tree]:
     """
-    Lark library doesn't have advance search function.
-    This function will return the first tree of provided rule
+    Lark library doesn't have an advance search function.
+    This function will return the first tree of the provided rule
     :param tree: Tree to search for the expression rule
     :return: Tree
     """
@@ -58,7 +58,7 @@ def internal(node: Union[Tree, Token]) -> Optional[Tree]:
         if isinstance(node, Token):
             return None
 
-        for child in cast(Tree, node).children:
+        for child in node.children:
             child_node: Optional[Tree] = internal(child)
             if child_node is not None:
                 return child_node
@@ -99,8 +99,7 @@ def internal(node: Union[Tree, Token]) -> None:
                 logger.debug(f"Unable to resolve parameter reference to {ref}")
                 values.append(ref)
         elif isinstance(node, Token):
-            # This means we're probably looking at a literal.
-            values.append(cast(Token, node).value)
+            values.append(node.value)
             return
         else:
             for child in node.children:
@@ -120,10 +119,14 @@ def remove_whitespaces_from_list(values: List[str]) -> List[str]:
     return result
 
 
+def strip_char(value: str, char: str = '"') -> str:
+    return value.strip(char)
+
+
 def strip_char_from_list(values: List[str], char: str = '"') -> List[str]:
     result: List[str] = []
     for item in values:
-        result.append(item.strip(char))
+        result.append(strip_char(item.strip(char), char=char))
 
     return result
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/validator.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/validator.py
index 9effdf5f97e30..b52977aaa41fb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/validator.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/validator.py
@@ -1,28 +1,34 @@
 import logging
-from typing import List, Optional, Tuple
+from typing import Optional, Tuple
 
-from lark import Tree
-
-from datahub.ingestion.source.powerbi.m_query import resolver, tree_function
+import datahub.ingestion.source.powerbi.m_query.data_classes
 
 logger = logging.getLogger(__name__)
 
 
 def validate_parse_tree(
-    tree: Tree, native_query_enabled: bool = True
+    expression: str, native_query_enabled: bool = True
 ) -> Tuple[bool, Optional[str]]:
     """
-    :param tree: tree to validate as per functions supported by m_parser module
+    :param expression: M-Query expression to check if supported data-function is present in expression
     :param native_query_enabled: Whether user want to extract lineage from native query
-    :return: first argument is False if validation is failed and second argument would contain the error message.
-             in the case of valid tree, the first argument is True and the second argument would be None.
+    :return: True or False.
     """
-    functions: List[str] = tree_function.get_all_function_name(tree)
-    if len(functions) == 0:
-        return False, "Function calls not found"
+    function_names = [
+        fun.value
+        for fun in datahub.ingestion.source.powerbi.m_query.data_classes.FunctionName
+    ]
+    if not any(fun in expression for fun in function_names):
+        return False, "DataAccess function is not present in M-Query expression."
 
     if native_query_enabled is False:
-        if resolver.FunctionName.NATIVE_QUERY.value in functions:
-            return False, "Lineage extraction from native query is disabled."
+        if (
+            datahub.ingestion.source.powerbi.m_query.data_classes.FunctionName.NATIVE_QUERY.value
+            in function_names
+        ):
+            return (
+                False,
+                "Lineage extraction from native query is disabled. Enable native_query_parsing in recipe",
+            )
 
     return True, None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi-lexical-grammar.rule b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi-lexical-grammar.rule
index 3524f43b25af7..f237e2503317f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi-lexical-grammar.rule
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi-lexical-grammar.rule
@@ -6,6 +6,26 @@
 //   This is necessary because unary_expression is the base for the
 //   whole relational_expression parse tree.
 // - Added letter_character_and_decimal_digit phrase and updated keyword_or_identifier phrase
+// - Added below pattern in argument_list
+//          |   WS_INLINE? sql_string
+//          |   WS_INLINE? sql_string "," argument_list
+// - Added subtract_expression
+// - Updated relational_expression, here below are the updates
+//      |   subtract_expression
+//      |   subtract_expression "<" relational_expression
+//      |   subtract_expression ">" relational_expression
+//      |   subtract_expression "<=" relational_expression
+//      |   subtract_expression ">=" relational_expression
+// - Added empty_string
+// - Updated argument_list, below are the updates
+//      |   empty_string
+//      |   empty_string "," argument_list
+// - Added sql_string in any_literal
+// - Added WS_INLINE? in field expression
+//   Added to ignore any comments
+//    %ignore WS             // Ignore whitespace
+//    %ignore CPP_COMMENT    // Ignore single-line comments
+//    %ignore C_COMMENT      // Ignore multi-line comments
 
 lexical_unit:   lexical_elements?
 
@@ -121,6 +141,7 @@ verbatim_literal: "#!\"" text_literal_characters? "\""
 identifier: regular_identifier
       |     quoted_identifier
 
+
 regular_identifier:     available_identifier
                   |     available_identifier dot_character regular_identifier
 
@@ -229,6 +250,8 @@ operator_or_punctuator: ","
                     |   "=>" 
                     |   ".." 
                     |   "..."
+                    |   "{{"
+                    |   "}}"
 
 document:   section_document
       |     expression_document
@@ -259,13 +282,16 @@ expression: logical_or_expression
       |     if_expression
       |     error_raising_expression
       |     error_handling_expression
+      |     outer_expression
+
 
 logical_or_expression:  logical_and_expression
                     |   logical_and_expression "or" logical_or_expression
 
+
 logical_and_expression: WS_INLINE? NEWLINE? is_expression WS_INLINE? NEWLINE?
                     |   WS_INLINE?
-                    |   NEWLINE?     
+                    |   NEWLINE?
                     |   logical_and_expression WS_INLINE? "and" WS_INLINE? is_expression
 
 is_expression:  as_expression
@@ -284,10 +310,16 @@ equality_expression:    WS_INLINE? relational_expression
                    |    relational_expression WS_INLINE? "<>" WS_INLINE? equality_expression
 
 relational_expression:  additive_expression
+                    |   subtract_expression
                     |   additive_expression "<" relational_expression
                     |   additive_expression ">" relational_expression
                     |   additive_expression "<=" relational_expression
                     |   additive_expression ">=" relational_expression
+                    |   subtract_expression "<" relational_expression
+                    |   subtract_expression ">" relational_expression
+                    |   subtract_expression "<=" relational_expression
+                    |   subtract_expression ">=" relational_expression
+
 
 additive_expression:    multiplicative_expression
                     |   multiplicative_expression "+" additive_expression
@@ -295,6 +327,11 @@ additive_expression:    multiplicative_expression
                     |   multiplicative_expression  WS_INLINE? NEWLINE? WS_INLINE? "&" WS_INLINE? NEWLINE? WS_INLINE? additive_expression
 
 
+subtract_expression:    multiplicative_expression
+                    |   multiplicative_expression "-" additive_expression
+                    |   multiplicative_expression WS_INLINE? "_" WS_INLINE? additive_expression
+                    |   multiplicative_expression  WS_INLINE? NEWLINE? WS_INLINE? "&" WS_INLINE? NEWLINE? WS_INLINE? additive_expression
+
 multiplicative_expression:  WS_INLINE? metadata_expression
                         |   metadata_expression WS_INLINE? "*" WS_INLINE? multiplicative_expression
                         |   metadata_expression "/" multiplicative_expression
@@ -340,10 +377,25 @@ not_implemented_expression: "..."
 
 invoke_expression:  "#"? primary_expression "(" NEWLINE? argument_list? NEWLINE? ")"
 
+empty_string: /"([^"]|\\")*"/
+
+// SQL String specific rules
+sql_content: /(?:[^\"\\]|\\[\"]|\"\"|\#\(lf\))+/
+
+sql_string: "\"" sql_content "\""
+
+outer_expression: "{{" expression "}}"
+
 argument_list:  WS_INLINE? expression
             |   WS_INLINE? expression WS_INLINE? "," WS_INLINE? argument_list
+            |   WS_INLINE? sql_string
+            |   WS_INLINE? sql_string "," argument_list
             |   "\"" identifier "\""
             |   "\"" identifier "\"" "," argument_list
+            |   "[" identifier "]"
+            |   "[" identifier "]" "," argument_list
+            |   empty_string
+            |   empty_string "," argument_list
             |   WS_INLINE
             |   WS_INLINE? ESCAPED_STRING
             |   WS_INLINE? ESCAPED_STRING "," argument_list
@@ -355,6 +407,7 @@ argument_list:  WS_INLINE? expression
 
 list_expression:    "{" item_list? "}"
 
+
 item_list:  item
         |   item "," item_list
 
@@ -366,7 +419,7 @@ record_expression:  "[" field_list? "]"
 field_list: field
         |   field "," field_list
 
-field:  field_name WS_INLINE? "=" WS_INLINE? expression
+field:  WS_INLINE? field_name WS_INLINE? "=" WS_INLINE? expression
 
 field_name: generalized_identifier
         |   quoted_identifier
@@ -565,6 +618,7 @@ any_literal:      record_literal
             |     number_literal
             |     text_literal
             |     null_literal
+            |     sql_string
 
 
 %import common.WORD
@@ -577,4 +631,8 @@ any_literal:      record_literal
 %import common.DIGIT
 %import common.LF
 %import common.CR
-%import common.ESCAPED_STRING
\ No newline at end of file
+%import common.ESCAPED_STRING
+
+%ignore WS            // Ignore whitespace
+%ignore CPP_COMMENT    // Ignore single-line comments
+%ignore C_COMMENT      // Ignore multi-line comments
\ No newline at end of file
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
index 065bbac9e9645..044946a5d308d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
@@ -4,9 +4,13 @@
 #
 #########################################################
 import logging
+from datetime import datetime
 from typing import Iterable, List, Optional, Tuple, Union
 
+import more_itertools
+
 import datahub.emitter.mce_builder as builder
+import datahub.ingestion.source.powerbi.m_query.data_classes
 import datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes as powerbi_data_classes
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import ContainerKey, gen_containers
@@ -34,18 +38,18 @@
 from datahub.ingestion.source.common.subtypes import (
     BIAssetSubTypes,
     BIContainerSubTypes,
-    DatasetSubTypes,
 )
 from datahub.ingestion.source.powerbi.config import (
     Constant,
     PowerBiDashboardSourceConfig,
     PowerBiDashboardSourceReport,
+    SupportedDataPlatform,
 )
 from datahub.ingestion.source.powerbi.dataplatform_instance_resolver import (
     AbstractDataPlatformInstanceResolver,
     create_dataplatform_instance_resolver,
 )
-from datahub.ingestion.source.powerbi.m_query import parser, resolver
+from datahub.ingestion.source.powerbi.m_query import parser
 from datahub.ingestion.source.powerbi.rest_api_wrapper.powerbi_api import PowerBiAPI
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
@@ -60,6 +64,7 @@
     FineGrainedLineageUpstreamType,
 )
 from datahub.metadata.schema_classes import (
+    AuditStampClass,
     BrowsePathsClass,
     ChangeTypeClass,
     ChartInfoClass,
@@ -71,6 +76,7 @@
     DatasetLineageTypeClass,
     DatasetProfileClass,
     DatasetPropertiesClass,
+    EdgeClass,
     GlobalTagsClass,
     OtherSchemaClass,
     OwnerClass,
@@ -142,9 +148,7 @@ def assets_urn_to_lowercase(self, value):
 
     def new_mcp(
         self,
-        entity_type,
         entity_urn,
-        aspect_name,
         aspect,
         change_type=ChangeTypeClass.UPSERT,
     ):
@@ -152,10 +156,8 @@ def new_mcp(
         Create MCP
         """
         return MetadataChangeProposalWrapper(
-            entityType=entity_type,
             changeType=change_type,
             entityUrn=entity_urn,
-            aspectName=aspect_name,
             aspect=aspect,
         )
 
@@ -176,15 +178,15 @@ def extract_dataset_schema(
     ) -> List[MetadataChangeProposalWrapper]:
         schema_metadata = self.to_datahub_schema(table)
         schema_mcp = self.new_mcp(
-            entity_type=Constant.DATASET,
             entity_urn=ds_urn,
-            aspect_name=Constant.SCHEMA_METADATA,
             aspect=schema_metadata,
         )
         return [schema_mcp]
 
     def make_fine_grained_lineage_class(
-        self, lineage: resolver.Lineage, dataset_urn: str
+        self,
+        lineage: datahub.ingestion.source.powerbi.m_query.data_classes.Lineage,
+        dataset_urn: str,
     ) -> List[FineGrainedLineage]:
         fine_grained_lineages: List[FineGrainedLineage] = []
 
@@ -236,7 +238,9 @@ def extract_lineage(
         upstream: List[UpstreamClass] = []
         cll_lineage: List[FineGrainedLineage] = []
 
-        upstream_lineage: List[resolver.Lineage] = parser.get_upstream_tables(
+        upstream_lineage: List[
+            datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+        ] = parser.get_upstream_tables(
             table=table,
             reporter=self.__reporter,
             platform_instance_resolver=self.__dataplatform_instance_resolver,
@@ -409,9 +413,7 @@ def to_datahub_dataset(
                     viewLanguage="m_query",
                 )
                 view_prop_mcp = self.new_mcp(
-                    entity_type=Constant.DATASET,
                     entity_urn=ds_urn,
-                    aspect_name=Constant.VIEW_PROPERTIES,
                     aspect=view_properties,
                 )
                 dataset_mcps.extend([view_prop_mcp])
@@ -425,30 +427,23 @@ def to_datahub_dataset(
             )
 
             info_mcp = self.new_mcp(
-                entity_type=Constant.DATASET,
                 entity_urn=ds_urn,
-                aspect_name=Constant.DATASET_PROPERTIES,
                 aspect=ds_properties,
             )
 
             # Remove status mcp
             status_mcp = self.new_mcp(
-                entity_type=Constant.DATASET,
                 entity_urn=ds_urn,
-                aspect_name=Constant.STATUS,
                 aspect=StatusClass(removed=False),
             )
             if self.__config.extract_dataset_schema:
                 dataset_mcps.extend(self.extract_dataset_schema(table, ds_urn))
 
             subtype_mcp = self.new_mcp(
-                entity_type=Constant.DATASET,
                 entity_urn=ds_urn,
-                aspect_name=Constant.SUBTYPES,
                 aspect=SubTypesClass(
                     typeNames=[
-                        DatasetSubTypes.POWERBI_DATASET_TABLE,
-                        DatasetSubTypes.VIEW,
+                        BIContainerSubTypes.POWERBI_DATASET_TABLE,
                     ]
                 ),
             )
@@ -464,9 +459,7 @@ def to_datahub_dataset(
                 # Dashboard owner MCP
                 ownership = OwnershipClass(owners=[owner_class])
                 owner_mcp = self.new_mcp(
-                    entity_type=Constant.DATASET,
                     entity_urn=ds_urn,
-                    aspect_name=Constant.OWNERSHIP,
                     aspect=ownership,
                 )
                 dataset_mcps.extend([owner_mcp])
@@ -606,17 +599,13 @@ def tile_custom_properties(tile: powerbi_data_classes.Tile) -> dict:
         )
 
         info_mcp = self.new_mcp(
-            entity_type=Constant.CHART,
             entity_urn=chart_urn,
-            aspect_name=Constant.CHART_INFO,
             aspect=chart_info_instance,
         )
 
         # removed status mcp
         status_mcp = self.new_mcp(
-            entity_type=Constant.CHART,
             entity_urn=chart_urn,
-            aspect_name=Constant.STATUS,
             aspect=StatusClass(removed=False),
         )
 
@@ -633,18 +622,14 @@ def tile_custom_properties(tile: powerbi_data_classes.Tile) -> dict:
         # Explicitly emitting this aspect isn't necessary, but we do it here to ensure that
         # the old, bad data gets overwritten.
         chart_key_mcp = self.new_mcp(
-            entity_type=Constant.CHART,
             entity_urn=chart_urn,
-            aspect_name=Constant.CHART_KEY,
             aspect=ChartUrn.from_string(chart_urn).to_key_aspect(),
         )
 
         # Browse path
         browse_path = BrowsePathsClass(paths=[f"/powerbi/{workspace.name}"])
         browse_path_mcp = self.new_mcp(
-            entity_type=Constant.CHART,
             entity_urn=chart_urn,
-            aspect_name=Constant.BROWSERPATH,
             aspect=browse_path,
         )
         result_mcps = [
@@ -710,17 +695,13 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict:
         )
 
         info_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=Constant.DASHBOARD_INFO,
             aspect=dashboard_info_cls,
         )
 
         # removed status mcp
         removed_status_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=Constant.STATUS,
             aspect=StatusClass(removed=False),
         )
 
@@ -732,9 +713,7 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict:
 
         # Dashboard key
         dashboard_key_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=Constant.DASHBOARD_KEY,
             aspect=dashboard_key_cls,
         )
 
@@ -750,9 +729,7 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict:
             # Dashboard owner MCP
             ownership = OwnershipClass(owners=owners)
             owner_mcp = self.new_mcp(
-                entity_type=Constant.DASHBOARD,
                 entity_urn=dashboard_urn,
-                aspect_name=Constant.OWNERSHIP,
                 aspect=ownership,
             )
 
@@ -761,9 +738,7 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict:
             paths=[f"/{Constant.PLATFORM_NAME}/{dashboard.workspace_name}"]
         )
         browse_path_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=Constant.BROWSERPATH,
             aspect=browse_path,
         )
 
@@ -827,7 +802,12 @@ def generate_container_for_workspace(
         container_work_units = gen_containers(
             container_key=self.workspace_key,
             name=workspace.name,
-            sub_types=[BIContainerSubTypes.POWERBI_WORKSPACE],
+            sub_types=[workspace.type],
+            extra_properties={
+                "workspace_id": workspace.id,
+                "workspace_name": workspace.name,
+                "workspace_type": workspace.type,
+            },
         )
         return container_work_units
 
@@ -858,9 +838,7 @@ def append_tag_mcp(
     ) -> None:
         if self.__config.extract_endorsements_to_tags and tags:
             tags_mcp = self.new_mcp(
-                entity_type=entity_type,
                 entity_urn=entity_urn,
-                aspect_name=Constant.GLOBAL_TAGS,
                 aspect=self.transform_tags(tags),
             )
             list_of_mcps.append(tags_mcp)
@@ -883,9 +861,7 @@ def to_datahub_user(
         user_key = CorpUserKeyClass(username=user.id)
 
         user_key_mcp = self.new_mcp(
-            entity_type=Constant.CORP_USER,
             entity_urn=user_urn,
-            aspect_name=Constant.CORP_USER_KEY,
             aspect=user_key,
         )
 
@@ -1028,17 +1004,13 @@ def to_chart_mcps(
             )
 
             info_mcp = self.new_mcp(
-                entity_type=Constant.CHART,
                 entity_urn=chart_urn,
-                aspect_name=Constant.CHART_INFO,
                 aspect=chart_info_instance,
             )
 
             # removed status mcp
             status_mcp = self.new_mcp(
-                entity_type=Constant.CHART,
                 entity_urn=chart_urn,
-                aspect_name=Constant.STATUS,
                 aspect=StatusClass(removed=False),
             )
             # Subtype mcp
@@ -1050,11 +1022,11 @@ def to_chart_mcps(
             )
 
             # Browse path
-            browse_path = BrowsePathsClass(paths=[f"/powerbi/{workspace.name}"])
+            browse_path = BrowsePathsClass(
+                paths=[f"/{Constant.PLATFORM_NAME}/{workspace.name}"]
+            )
             browse_path_mcp = self.new_mcp(
-                entity_type=Constant.CHART,
                 entity_urn=chart_urn,
-                aspect_name=Constant.BROWSERPATH,
                 aspect=browse_path,
             )
             list_of_mcps = [info_mcp, status_mcp, subtype_mcp, browse_path_mcp]
@@ -1105,17 +1077,13 @@ def report_to_dashboard(
         )
 
         info_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=Constant.DASHBOARD_INFO,
             aspect=dashboard_info_cls,
         )
 
         # removed status mcp
         removed_status_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=Constant.STATUS,
             aspect=StatusClass(removed=False),
         )
 
@@ -1127,9 +1095,7 @@ def report_to_dashboard(
 
         # Dashboard key
         dashboard_key_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=Constant.DASHBOARD_KEY,
             aspect=dashboard_key_cls,
         )
         # Report Ownership
@@ -1144,9 +1110,7 @@ def report_to_dashboard(
             # Report owner MCP
             ownership = OwnershipClass(owners=owners)
             owner_mcp = self.new_mcp(
-                entity_type=Constant.DASHBOARD,
                 entity_urn=dashboard_urn,
-                aspect_name=Constant.OWNERSHIP,
                 aspect=ownership,
             )
 
@@ -1155,17 +1119,13 @@ def report_to_dashboard(
             paths=[f"/{Constant.PLATFORM_NAME}/{workspace.name}"]
         )
         browse_path_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=Constant.BROWSERPATH,
             aspect=browse_path,
         )
 
         sub_type_mcp = self.new_mcp(
-            entity_type=Constant.DASHBOARD,
             entity_urn=dashboard_urn,
-            aspect_name=SubTypesClass.ASPECT_NAME,
-            aspect=SubTypesClass(typeNames=[Constant.REPORT_TYPE_NAME]),
+            aspect=SubTypesClass(typeNames=[report.type.value]),
         )
 
         list_of_mcps = [
@@ -1203,7 +1163,7 @@ def report_to_datahub_work_units(
         logger.debug(f"Converting report={report.name} to datahub dashboard")
         # Convert user to CorpUser
         user_mcps = self.to_datahub_users(report.users)
-        # Convert pages to charts. A report has single dataset and same dataset used in pages to create visualization
+        # Convert pages to charts. A report has a single dataset and the same dataset used in pages to create visualization
         ds_mcps = self.to_datahub_dataset(report.dataset, workspace)
         chart_mcps = self.pages_to_chart(report.pages, workspace, ds_mcps)
 
@@ -1267,7 +1227,10 @@ def __init__(self, config: PowerBiDashboardSourceConfig, ctx: PipelineContext):
             self.source_config
         )
         try:
-            self.powerbi_client = PowerBiAPI(self.source_config)
+            self.powerbi_client = PowerBiAPI(
+                config=self.source_config,
+                reporter=self.reporter,
+            )
         except Exception as e:
             logger.warning(e)
             exit(
@@ -1288,7 +1251,10 @@ def __init__(self, config: PowerBiDashboardSourceConfig, ctx: PipelineContext):
     def test_connection(config_dict: dict) -> TestConnectionReport:
         test_report = TestConnectionReport()
         try:
-            PowerBiAPI(PowerBiDashboardSourceConfig.parse_obj_allow_extras(config_dict))
+            PowerBiAPI(
+                PowerBiDashboardSourceConfig.parse_obj_allow_extras(config_dict),
+                PowerBiDashboardSourceReport(),
+            )
             test_report.basic_connectivity = CapabilityReport(capable=True)
         except Exception as e:
             test_report.basic_connectivity = CapabilityReport(
@@ -1303,24 +1269,38 @@ def create(cls, config_dict, ctx):
 
     def get_allowed_workspaces(self) -> List[powerbi_data_classes.Workspace]:
         all_workspaces = self.powerbi_client.get_workspaces()
+        logger.info(f"Number of workspaces = {len(all_workspaces)}")
+        self.reporter.all_workspace_count = len(all_workspaces)
+        logger.debug(
+            f"All workspaces: {[workspace.format_name_for_logger() for workspace in all_workspaces]}"
+        )
 
-        allowed_wrk = [
-            workspace
-            for workspace in all_workspaces
-            if self.source_config.workspace_id_pattern.allowed(workspace.id)
-        ]
+        allowed_workspaces = []
+        for workspace in all_workspaces:
+            if not self.source_config.workspace_id_pattern.allowed(workspace.id):
+                self.reporter.filtered_workspace_names.append(
+                    f"{workspace.id} - {workspace.name}"
+                )
+                continue
+            elif workspace.type not in self.source_config.workspace_type_filter:
+                self.reporter.filtered_workspace_types.append(
+                    f"{workspace.id} - {workspace.name} (type = {workspace.type})"
+                )
+                continue
+            else:
+                allowed_workspaces.append(workspace)
 
-        logger.info(f"Number of workspaces = {len(all_workspaces)}")
-        self.reporter.report_number_of_workspaces(len(all_workspaces))
-        logger.info(f"Number of allowed workspaces = {len(allowed_wrk)}")
-        logger.debug(f"Workspaces = {all_workspaces}")
+        logger.info(f"Number of allowed workspaces = {len(allowed_workspaces)}")
+        logger.debug(
+            f"Allowed workspaces: {[workspace.format_name_for_logger() for workspace in allowed_workspaces]}"
+        )
 
-        return allowed_wrk
+        return allowed_workspaces
 
     def validate_dataset_type_mapping(self):
         powerbi_data_platforms: List[str] = [
             data_platform.value.powerbi_data_platform_name
-            for data_platform in resolver.SupportedDataPlatform
+            for data_platform in SupportedDataPlatform
         ]
 
         for key in self.source_config.dataset_type_mapping.keys():
@@ -1357,6 +1337,95 @@ def extract_independent_datasets(
                 )
             )
 
+    def emit_app(
+        self, workspace: powerbi_data_classes.Workspace
+    ) -> Iterable[MetadataChangeProposalWrapper]:
+        if workspace.app is None:
+            return
+
+        if not self.source_config.extract_app:
+            self.reporter.info(
+                title="App Ingestion Is Disabled",
+                message="You are missing workspace app metadata. Please set flag `extract_app` to `true` in recipe to ingest workspace app.",
+                context=f"workspace-name={workspace.name}, app-name = {workspace.app.name}",
+            )
+            return
+
+        assets_within_app: List[EdgeClass] = [
+            EdgeClass(
+                destinationUrn=builder.make_dashboard_urn(
+                    platform=self.source_config.platform_name,
+                    platform_instance=self.source_config.platform_instance,
+                    name=powerbi_data_classes.Dashboard.get_urn_part_by_id(
+                        app_dashboard.original_dashboard_id
+                    ),
+                )
+            )
+            for app_dashboard in workspace.app.dashboards
+        ]
+
+        assets_within_app.extend(
+            [
+                EdgeClass(
+                    destinationUrn=builder.make_dashboard_urn(
+                        platform=self.source_config.platform_name,
+                        platform_instance=self.source_config.platform_instance,
+                        name=powerbi_data_classes.Report.get_urn_part_by_id(
+                            app_report.original_report_id
+                        ),
+                    )
+                )
+                for app_report in workspace.app.reports
+            ]
+        )
+
+        if assets_within_app:
+            logger.debug(
+                f"Emitting metadata-workunits for app {workspace.app.name}({workspace.app.id})"
+            )
+
+            app_urn: str = builder.make_dashboard_urn(
+                platform=self.source_config.platform_name,
+                platform_instance=self.source_config.platform_instance,
+                name=powerbi_data_classes.App.get_urn_part_by_id(workspace.app.id),
+            )
+
+            dashboard_info: DashboardInfoClass = DashboardInfoClass(
+                title=workspace.app.name,
+                description=workspace.app.description
+                if workspace.app.description
+                else workspace.app.name,
+                # lastModified=workspace.app.last_update,
+                lastModified=ChangeAuditStamps(
+                    lastModified=AuditStampClass(
+                        actor="urn:li:corpuser:unknown",
+                        time=int(
+                            datetime.strptime(
+                                workspace.app.last_update, "%Y-%m-%dT%H:%M:%S.%fZ"
+                            ).timestamp()
+                        ),
+                    )
+                    if workspace.app.last_update
+                    else None
+                ),
+                dashboards=assets_within_app,
+            )
+
+            # Browse path
+            browse_path: BrowsePathsClass = BrowsePathsClass(
+                paths=[f"/powerbi/{workspace.name}"]
+            )
+
+            yield from MetadataChangeProposalWrapper.construct_many(
+                entityUrn=app_urn,
+                aspects=(
+                    dashboard_info,
+                    browse_path,
+                    StatusClass(removed=False),
+                    SubTypesClass(typeNames=[BIAssetSubTypes.POWERBI_APP]),
+                ),
+            )
+
     def get_workspace_workunit(
         self, workspace: powerbi_data_classes.Workspace
     ) -> Iterable[MetadataWorkUnit]:
@@ -1366,8 +1435,11 @@ def get_workspace_workunit(
             )
 
             for workunit in workspace_workunits:
-                # Return workunit to Datahub Ingestion framework
+                # Return workunit to a Datahub Ingestion framework
                 yield workunit
+
+        yield from auto_workunit(self.emit_app(workspace=workspace))
+
         for dashboard in workspace.dashboards:
             try:
                 # Fetch PowerBi users for dashboards
@@ -1415,7 +1487,7 @@ def _get_dashboard_patch_work_unit(
 
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         # As modified_workspaces is not idempotent, hence workunit processors are run later for each workspace_id
-        # This will result in creating checkpoint for each workspace_id
+        # This will result in creating a checkpoint for each workspace_id
         if self.source_config.modified_since:
             return []  # Handle these in get_workunits_internal
         else:
@@ -1426,7 +1498,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         """
-        Datahub Ingestion framework invoke this method
+        Datahub Ingestion framework invokes this method
         """
         logger.info("PowerBi plugin execution is started")
         # Validate dataset type mapping
@@ -1434,16 +1506,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         # Fetch PowerBi workspace for given workspace identifier
 
         allowed_workspaces = self.get_allowed_workspaces()
-        workspaces_len = len(allowed_workspaces)
-
-        batch_size = (
-            self.source_config.scan_batch_size
-        )  # 100 is the maximum allowed for powerbi scan
-        num_batches = (workspaces_len + batch_size - 1) // batch_size
-        batches = [
-            allowed_workspaces[i * batch_size : (i + 1) * batch_size]
-            for i in range(num_batches)
-        ]
+
+        batches = more_itertools.chunked(
+            allowed_workspaces, self.source_config.scan_batch_size
+        )
         for batch_workspaces in batches:
             for workspace in self.powerbi_client.fill_workspaces(
                 batch_workspaces, self.reporter
@@ -1452,7 +1518,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
                 if self.source_config.modified_since:
                     # As modified_workspaces is not idempotent, hence we checkpoint for each powerbi workspace
-                    # Because job_id is used as dictionary key, we have to set a new job_id
+                    # Because job_id is used as a dictionary key, we have to set a new job_id
                     # Refer to https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py#L390
                     self.stale_entity_removal_handler.set_job_id(workspace.id)
                     self.state_provider.register_stateful_ingestion_usecase_handler(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py
index 5106b9817d351..70e4ee68f5351 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py
@@ -37,10 +37,40 @@ class DatasetKey(ContainerKey):
     dataset: str
 
 
+@dataclass
+class AppDashboard:
+    id: str
+    original_dashboard_id: str
+
+
+@dataclass
+class AppReport:
+    id: str
+    original_report_id: str
+
+
+@dataclass
+class App:
+    id: str
+    name: str
+    description: Optional[str]
+    last_update: Optional[str]
+    dashboards: List["AppDashboard"]
+    reports: List["AppReport"]
+
+    def get_urn_part(self):
+        return App.get_urn_part_by_id(self.id)
+
+    @staticmethod
+    def get_urn_part_by_id(id_: str) -> str:
+        return f"apps.{id_}"
+
+
 @dataclass
 class Workspace:
     id: str
     name: str
+    type: str  # This is used as a subtype of the Container entity.
     dashboards: List["Dashboard"]
     reports: List["Report"]
     datasets: Dict[str, "PowerBIDataset"]
@@ -48,6 +78,7 @@ class Workspace:
     dashboard_endorsements: Dict[str, List[str]]
     scan_result: dict
     independent_datasets: List["PowerBIDataset"]
+    app: Optional["App"]
 
     def get_urn_part(self, workspace_id_as_urn_part: Optional[bool] = False) -> str:
         # shouldn't use workspace name, as they can be the same?
@@ -65,6 +96,9 @@ def get_workspace_key(
             instance=platform_instance,
         )
 
+    def format_name_for_logger(self) -> str:
+        return f"{self.name} ({self.id})"
+
 
 @dataclass
 class DataSource:
@@ -141,6 +175,7 @@ class PowerBIDataset:
     description: str
     webUrl: Optional[str]
     workspace_id: str
+    workspace_name: str
     parameters: Dict[str, str]
 
     # Table in datasets
@@ -194,7 +229,7 @@ class User:
     groupUserAccessRight: Optional[str] = None
 
     def get_urn_part(self, use_email: bool, remove_email_suffix: bool) -> str:
-        if use_email:
+        if use_email and self.emailAddress:
             if remove_email_suffix:
                 return self.emailAddress.split("@")[0]
             else:
@@ -211,20 +246,33 @@ def __hash__(self):
         return hash(self.__members())
 
 
+class ReportType(Enum):
+    PaginatedReport = "PaginatedReport"
+    PowerBIReport = "Report"
+
+
 @dataclass
 class Report:
     id: str
     name: str
+    type: ReportType
     webUrl: Optional[str]
     embedUrl: str
     description: str
-    dataset: Optional["PowerBIDataset"]
+    dataset_id: Optional[str]  # dataset_id is coming from REST API response
+    dataset: Optional[
+        "PowerBIDataset"
+    ]  # This the dataclass later initialise by powerbi_api.py
     pages: List["Page"]
     users: List["User"]
     tags: List[str]
 
     def get_urn_part(self):
-        return f"reports.{self.id}"
+        return Report.get_urn_part_by_id(self.id)
+
+    @staticmethod
+    def get_urn_part_by_id(id_: str) -> str:
+        return f"reports.{id_}"
 
 
 @dataclass
@@ -259,10 +307,14 @@ class Dashboard:
     tiles: List["Tile"]
     users: List["User"]
     tags: List[str]
-    webUrl: Optional[str] = None
+    webUrl: Optional[str]
 
     def get_urn_part(self):
-        return f"dashboards.{self.id}"
+        return Dashboard.get_urn_part_by_id(self.id)
+
+    @staticmethod
+    def get_urn_part_by_id(id_: str) -> str:
+        return f"dashboards.{id_}"
 
     def __members(self):
         return (self.id,)
@@ -276,7 +328,7 @@ def __hash__(self):
         return hash(self.__members())
 
 
-def new_powerbi_dataset(workspace_id: str, raw_instance: dict) -> PowerBIDataset:
+def new_powerbi_dataset(workspace: Workspace, raw_instance: dict) -> PowerBIDataset:
     return PowerBIDataset(
         id=raw_instance["id"],
         name=raw_instance.get("name"),
@@ -286,7 +338,8 @@ def new_powerbi_dataset(workspace_id: str, raw_instance: dict) -> PowerBIDataset
             if raw_instance.get("webUrl") is not None
             else None
         ),
-        workspace_id=workspace_id,
+        workspace_id=workspace.id,
+        workspace_name=workspace.name,
         parameters={},
         tables=[],
         tags=[],
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py
index b190cf065b6e3..e1301edef10b8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py
@@ -1,9 +1,8 @@
 import logging
-import math
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta
 from time import sleep
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Iterator, List, Optional, Union
 
 import msal
 import requests
@@ -14,6 +13,7 @@
 from datahub.configuration.common import AllowDenyPattern, ConfigurationError
 from datahub.ingestion.source.powerbi.config import Constant
 from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import (
+    App,
     Column,
     Dashboard,
     Measure,
@@ -21,6 +21,7 @@
     Page,
     PowerBIDataset,
     Report,
+    ReportType,
     Table,
     Tile,
     User,
@@ -55,9 +56,23 @@ def is_http_failure(response: Response, message: str) -> bool:
     return True
 
 
+class SessionWithTimeout(requests.Session):
+    timeout: int
+
+    def __init__(self, timeout, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.timeout = timeout
+
+    def request(self, method, url, **kwargs):
+        # Set the default timeout if none is provided
+        kwargs.setdefault("timeout", self.timeout)
+        return super().request(method, url, **kwargs)
+
+
 class DataResolverBase(ABC):
     SCOPE: str = "https://analysis.windows.net/powerbi/api/.default"
-    BASE_URL: str = "https://api.powerbi.com/v1.0/myorg/groups"
+    MY_ORG_URL = "https://api.powerbi.com/v1.0/myorg"
+    BASE_URL: str = f"{MY_ORG_URL}/groups"
     ADMIN_BASE_URL: str = "https://api.powerbi.com/v1.0/myorg/admin"
     AUTHORITY: str = "https://login.microsoftonline.com/"
     TOP: int = 1000
@@ -67,6 +82,7 @@ def __init__(
         client_id: str,
         client_secret: str,
         tenant_id: str,
+        metadata_api_timeout: int,
     ):
         self.__access_token: Optional[str] = None
         self.__access_token_expiry_time: Optional[datetime] = None
@@ -82,7 +98,9 @@ def __init__(
         self.get_access_token()
 
         logger.info(f"Connected to {self._get_authority_url()}")
-        self._request_session = requests.Session()
+
+        self._request_session = SessionWithTimeout(timeout=metadata_api_timeout)
+
         # set re-try parameter for request_session
         self._request_session.mount(
             "https://",
@@ -128,7 +146,7 @@ def profile_dataset(
 
     @abstractmethod
     def get_dataset(
-        self, workspace_id: str, dataset_id: str
+        self, workspace: Workspace, dataset_id: str
     ) -> Optional[PowerBIDataset]:
         pass
 
@@ -142,6 +160,13 @@ def get_dataset_parameters(
     def get_users(self, workspace_id: str, entity: str, entity_id: str) -> List[User]:
         pass
 
+    @abstractmethod
+    def _get_app(
+        self,
+        app_id: str,
+    ) -> Optional[Dict]:
+        pass
+
     def _get_authority_url(self):
         return f"{DataResolverBase.AUTHORITY}{self.__tenant_id}"
 
@@ -222,49 +247,27 @@ def get_dashboards(self, workspace: Workspace) -> List[Dashboard]:
                 tags=[],
             )
             for instance in dashboards_dict
-            if instance is not None
+            if (
+                instance is not None
+                and Constant.APP_ID
+                not in instance  # As we add dashboards to the App, Power BI starts
+                # providing duplicate dashboard information,
+                # where the duplicate includes an AppId, while the original dashboard does not.
+            )
         ]
 
         return dashboards
 
-    def get_groups(self) -> List[dict]:
+    def get_groups(self, filter_: Dict) -> List[dict]:
         group_endpoint = self.get_groups_endpoint()
-        params: dict = {"$top": self.TOP, "$skip": 0, "$filter": "type eq 'Workspace'"}
 
-        def fetch_page(page_number: int) -> dict:
-            params["$skip"] = self.TOP * page_number
-            logger.debug(f"Query parameters = {params}")
-            response = self._request_session.get(
-                group_endpoint,
-                headers=self.get_authorization_header(),
-                params=params,
-            )
-            response.raise_for_status()
-            return response.json()
-
-        # Hit PowerBi
-        logger.debug(f"Request to groups endpoint URL={group_endpoint}")
-        zeroth_page = fetch_page(0)
-        logger.debug(f"Page 0 = {zeroth_page}")
-        if zeroth_page.get(Constant.ODATA_COUNT) is None:
-            logger.warning(
-                "@odata.count field is not present in response. Unable to fetch workspaces."
-            )
-            return []
-
-        number_of_items = zeroth_page[Constant.ODATA_COUNT]
-        number_of_pages = math.ceil(number_of_items / self.TOP)
-        output: List[dict] = zeroth_page[Constant.VALUE]
-        for page in range(
-            1, number_of_pages
-        ):  # start from 1 as 0th index already fetched
-            page_response = fetch_page(page)
-            if len(page_response[Constant.VALUE]) == 0:
-                break
-
-            logger.debug(f"Page {page} = {zeroth_page}")
+        output: List[dict] = []
 
-            output.extend(page_response[Constant.VALUE])
+        for page in self.itr_pages(
+            endpoint=group_endpoint,
+            parameter_override=filter_,
+        ):
+            output.extend(page)
 
         return output
 
@@ -286,24 +289,31 @@ def fetch_reports():
             )
             response.raise_for_status()
             response_dict = response.json()
-            logger.debug(f"Request response = {response_dict}")
+            logger.debug(f"Report Request response = {response_dict}")
             return response_dict.get(Constant.VALUE, [])
 
         reports: List[Report] = [
             Report(
                 id=raw_instance.get(Constant.ID),
                 name=raw_instance.get(Constant.NAME),
+                type=ReportType[raw_instance.get(Constant.REPORT_TYPE)],
                 webUrl=raw_instance.get(Constant.WEB_URL),
                 embedUrl=raw_instance.get(Constant.EMBED_URL),
                 description=raw_instance.get(Constant.DESCRIPTION, ""),
                 pages=self._get_pages_by_report(
                     workspace=workspace, report_id=raw_instance[Constant.ID]
                 ),
+                dataset_id=raw_instance.get(Constant.DATASET_ID),
                 users=[],  # It will be fetched using Admin Fetcher based on condition
                 tags=[],  # It will be fetched using Admin Fetcher based on condition
-                dataset=workspace.datasets.get(raw_instance.get(Constant.DATASET_ID)),
+                dataset=None,  # It will come from dataset_registry defined in powerbi_api.py
             )
             for raw_instance in fetch_reports()
+            if Constant.APP_ID
+            not in raw_instance  # As we add reports to the App, Power BI starts providing
+            # duplicate report information,
+            # where the duplicate includes an AppId,
+            # while the original report does not.
         ]
 
         return reports
@@ -332,11 +342,6 @@ def new_dataset_or_report(tile_instance: Any) -> dict:
             Find out which is the data source for tile. It is either REPORT or DATASET
             """
             report_fields = {
-                Constant.DATASET: (
-                    workspace.datasets.get(tile_instance.get(Constant.DATASET_ID))
-                    if tile_instance.get("datasetId") is not None
-                    else None
-                ),
                 Constant.REPORT: (
                     self.get_report(
                         workspace=workspace,
@@ -387,6 +392,7 @@ def new_dataset_or_report(tile_instance: Any) -> dict:
                 title=instance.get(Constant.TITLE),
                 embedUrl=instance.get(Constant.EMBED_URL),
                 dataset_id=instance.get(Constant.DATASET_ID),
+                dataset=None,
                 **new_dataset_or_report(instance),
             )
             for instance in tile_dict
@@ -395,6 +401,70 @@ def new_dataset_or_report(tile_instance: Any) -> dict:
 
         return tiles
 
+    def itr_pages(
+        self,
+        endpoint: str,
+        parameter_override: Dict = {},
+    ) -> Iterator[List[Dict]]:
+        params: dict = {
+            "$skip": 0,
+            "$top": self.TOP,
+            **parameter_override,
+        }
+
+        page_number: int = 0
+
+        while True:
+            params["$skip"] = self.TOP * page_number
+            response = self._request_session.get(
+                endpoint,
+                headers=self.get_authorization_header(),
+                params=params,
+            )
+
+            response.raise_for_status()
+
+            assert (
+                Constant.VALUE in response.json()
+            ), "'value' key is not present in paginated response"
+
+            if not response.json()[Constant.VALUE]:  # if it is an empty list then break
+                break
+
+            yield response.json()[Constant.VALUE]
+
+            page_number += 1
+
+    def get_app(
+        self,
+        app_id: str,
+    ) -> Optional[App]:
+        raw_app: Optional[Dict] = self._get_app(
+            app_id=app_id,
+        )
+
+        if raw_app is None:
+            return None
+
+        assert (
+            Constant.ID in raw_app
+        ), f"{Constant.ID} is required field not present in server response"
+
+        assert (
+            Constant.NAME in raw_app
+        ), f"{Constant.NAME} is required field not present in server response"
+
+        return App(
+            id=raw_app[Constant.ID],
+            name=raw_app[Constant.NAME],
+            description=raw_app.get(Constant.DESCRIPTION),
+            last_update=raw_app.get(Constant.LAST_UPDATE),
+            dashboards=[],  # dashboards and reports of App are available in scan-result response
+            reports=[],  # There is an App section in documentation https://learn.microsoft.com/en-us/rest/api/power-bi/dashboards/get-dashboards-in-group#code-try-0
+            # However the report API mentioned in that section is not returning the reports
+            # We will collect these details from the scan-result.
+        )
+
 
 class RegularAPIResolver(DataResolverBase):
     # Regular access endpoints
@@ -407,17 +477,18 @@ class RegularAPIResolver(DataResolverBase):
         Constant.REPORT_LIST: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports",
         Constant.PAGE_BY_REPORT: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports/{REPORT_ID}/pages",
         Constant.DATASET_EXECUTE_QUERIES: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/datasets/{DATASET_ID}/executeQueries",
+        Constant.GET_WORKSPACE_APP: "{MY_ORG_URL}/apps/{APP_ID}",
     }
 
     def get_dataset(
-        self, workspace_id: str, dataset_id: str
+        self, workspace: Workspace, dataset_id: str
     ) -> Optional[PowerBIDataset]:
         """
         Fetch the dataset from PowerBi for the given dataset identifier
         """
-        if workspace_id is None or dataset_id is None:
+        if workspace.id is None or dataset_id is None:
             logger.debug("Input values are None")
-            logger.debug(f"{Constant.WorkspaceId}={workspace_id}")
+            logger.debug(f"{Constant.WorkspaceId}={workspace.id}")
             logger.debug(f"{Constant.DatasetId}={dataset_id}")
             return None
 
@@ -427,7 +498,7 @@ def get_dataset(
         # Replace place holders
         dataset_get_endpoint = dataset_get_endpoint.format(
             POWERBI_BASE_URL=DataResolverBase.BASE_URL,
-            WORKSPACE_ID=workspace_id,
+            WORKSPACE_ID=workspace.id,
             DATASET_ID=dataset_id,
         )
         # Hit PowerBi
@@ -436,13 +507,13 @@ def get_dataset(
             dataset_get_endpoint,
             headers=self.get_authorization_header(),
         )
-        # Check if we got response from PowerBi
+        # Check if we got a response from PowerBi
         response.raise_for_status()
         response_dict = response.json()
         logger.debug(f"datasets = {response_dict}")
-        # PowerBi Always return the webURL, in-case if it is None then setting complete webURL to None instead of
+        # PowerBi Always return the webURL, in-case if it is None, then setting complete webURL to None instead of
         # None/details
-        return new_powerbi_dataset(workspace_id, response_dict)
+        return new_powerbi_dataset(workspace, response_dict)
 
     def get_dataset_parameters(
         self, workspace_id: str, dataset_id: str
@@ -663,6 +734,15 @@ def profile_dataset(
 
         table.column_count = column_count
 
+    def _get_app(
+        self,
+        app_id: str,
+    ) -> Optional[Dict]:
+        # [Date: 2024/10/18] As per API doc, the service principal approach is not supported for regular API
+        # https://learn.microsoft.com/en-us/rest/api/power-bi/apps/get-app
+
+        return None
+
 
 class AdminAPIResolver(DataResolverBase):
     # Admin access endpoints
@@ -676,6 +756,7 @@ class AdminAPIResolver(DataResolverBase):
         Constant.ENTITY_USER_LIST: "{POWERBI_ADMIN_BASE_URL}/{ENTITY}/{ENTITY_ID}/users",
         Constant.DATASET_LIST: "{POWERBI_ADMIN_BASE_URL}/groups/{WORKSPACE_ID}/datasets",
         Constant.WORKSPACE_MODIFIED_LIST: "{POWERBI_ADMIN_BASE_URL}/workspaces/modified",
+        Constant.GET_WORKSPACE_APP: "{POWERBI_ADMIN_BASE_URL}/apps",
     }
 
     def create_scan_job(self, workspace_ids: List[str]) -> str:
@@ -745,7 +826,7 @@ def _is_scan_result_ready(
             res.raise_for_status()
 
             if res.json()[Constant.STATUS].upper() == Constant.SUCCEEDED:
-                logger.info(f"Scan result is available for scan id({scan_id})")
+                logger.debug(f"Scan result is available for scan id({scan_id})")
                 return True
 
             if retry == max_retry:
@@ -832,8 +913,8 @@ def get_users(self, workspace_id: str, entity: str, entity_id: str) -> List[User
         return users
 
     def get_scan_result(self, scan_id: str) -> Optional[dict]:
-        logger.info("Fetching scan result")
-        logger.info(f"{Constant.SCAN_ID}={scan_id}")
+        logger.debug("Fetching scan result")
+        logger.debug(f"{Constant.SCAN_ID}={scan_id}")
         scan_result_get_endpoint = AdminAPIResolver.API_ENDPOINTS[
             Constant.SCAN_RESULT_GET
         ]
@@ -889,11 +970,11 @@ def get_tiles_endpoint(self, workspace: Workspace, dashboard_id: str) -> str:
         )
 
     def get_dataset(
-        self, workspace_id: str, dataset_id: str
+        self, workspace: Workspace, dataset_id: str
     ) -> Optional[PowerBIDataset]:
         datasets_endpoint = self.API_ENDPOINTS[Constant.DATASET_LIST].format(
             POWERBI_ADMIN_BASE_URL=DataResolverBase.ADMIN_BASE_URL,
-            WORKSPACE_ID=workspace_id,
+            WORKSPACE_ID=workspace.id,
         )
         # Hit PowerBi
         logger.debug(f"Request to datasets URL={datasets_endpoint}")
@@ -909,20 +990,20 @@ def get_dataset(
         if len(response_dict.get(Constant.VALUE, [])) == 0:
             logger.warning(
                 "Dataset not found. workspace_id = %s, dataset_id = %s",
-                workspace_id,
+                workspace.id,
                 dataset_id,
             )
             return None
 
         raw_instance: dict = response_dict[Constant.VALUE][0]
-        return new_powerbi_dataset(workspace_id, raw_instance)
+        return new_powerbi_dataset(workspace, raw_instance)
 
     def _get_pages_by_report(self, workspace: Workspace, report_id: str) -> List[Page]:
         return []  # Report pages are not available in Admin API
 
     def get_modified_workspaces(self, modified_since: str) -> List[str]:
         """
-        Get list of modified workspaces
+        Get a list of modified workspaces
         """
         modified_workspaces_endpoint = self.API_ENDPOINTS[
             Constant.WORKSPACE_MODIFIED_LIST
@@ -930,7 +1011,7 @@ def get_modified_workspaces(self, modified_since: str) -> List[str]:
             POWERBI_ADMIN_BASE_URL=DataResolverBase.ADMIN_BASE_URL,
         )
         parameters: Dict[str, Any] = {
-            "excludePersonalWorkspaces": True,
+            "excludePersonalWorkspaces": False,
             "excludeInActiveWorkspaces": True,
             "modifiedSince": modified_since,
         }
@@ -975,3 +1056,21 @@ def profile_dataset(
     ) -> None:
         logger.debug("Profile dataset is unsupported in Admin API")
         return None
+
+    def _get_app(
+        self,
+        app_id: str,
+    ) -> Optional[Dict]:
+        app_endpoint = self.API_ENDPOINTS[Constant.GET_WORKSPACE_APP].format(
+            POWERBI_ADMIN_BASE_URL=DataResolverBase.ADMIN_BASE_URL,
+            APP_ID=app_id,
+        )
+        # Hit PowerBi
+        logger.debug(f"Request to app URL={app_endpoint}")
+
+        for page in self.itr_pages(endpoint=app_endpoint):
+            for app in page:
+                if Constant.ID in app and app_id == app[Constant.ID]:
+                    return app
+
+        return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py
index a245d4c2b9a35..5ae333430a78b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py
@@ -13,6 +13,9 @@
 from datahub.ingestion.source.powerbi.rest_api_wrapper import data_resolver
 from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import (
     FIELD_TYPE_MAPPING,
+    App,
+    AppDashboard,
+    AppReport,
     Column,
     Dashboard,
     Measure,
@@ -31,28 +34,72 @@
 logger = logging.getLogger(__name__)
 
 
+def form_full_table_name(
+    config: PowerBiDashboardSourceConfig,
+    workspace: Workspace,
+    dataset_name: str,
+    table_name: str,
+) -> str:
+    full_table_name: str = "{}.{}".format(
+        dataset_name.replace(" ", "_"), table_name.replace(" ", "_")
+    )
+
+    if config.include_workspace_name_in_dataset_urn:
+        workspace_identifier: str = (
+            workspace.id
+            if config.workspace_id_as_urn_part
+            else workspace.name.replace(" ", "_").lower()
+        )
+        full_table_name = f"{workspace_identifier}.{full_table_name}"
+
+    return full_table_name
+
+
 class PowerBiAPI:
-    def __init__(self, config: PowerBiDashboardSourceConfig) -> None:
+    def __init__(
+        self,
+        config: PowerBiDashboardSourceConfig,
+        reporter: PowerBiDashboardSourceReport,
+    ) -> None:
         self.__config: PowerBiDashboardSourceConfig = config
+        self.__reporter = reporter
 
         self.__regular_api_resolver = RegularAPIResolver(
             client_id=self.__config.client_id,
             client_secret=self.__config.client_secret,
             tenant_id=self.__config.tenant_id,
+            metadata_api_timeout=self.__config.metadata_api_timeout,
         )
 
         self.__admin_api_resolver = AdminAPIResolver(
             client_id=self.__config.client_id,
             client_secret=self.__config.client_secret,
             tenant_id=self.__config.tenant_id,
+            metadata_api_timeout=self.__config.metadata_api_timeout,
         )
 
+        self.reporter: PowerBiDashboardSourceReport = reporter
+
+        # A report or tile in one workspace can be built using a dataset from another workspace.
+        # We need to store the dataset ID (which is a UUID) mapped to its dataset instance.
+        # This mapping will allow us to retrieve the appropriate dataset for
+        # reports and tiles across different workspaces.
+        self.dataset_registry: Dict[str, PowerBIDataset] = {}
+
     def log_http_error(self, message: str) -> Any:
         logger.warning(message)
         _, e, _ = sys.exc_info()
         if isinstance(e, requests.exceptions.HTTPError):
             logger.warning(f"HTTP status-code = {e.response.status_code}")
 
+        if isinstance(e, requests.exceptions.Timeout):
+            url: str = e.request.url if e.request else "URL not available"
+            self.reporter.warning(
+                title="Metadata API Timeout",
+                message=f"Metadata endpoints are not reachable. Check network connectivity to PowerBI Service.",
+                context=f"url={url}",
+            )
+
         logger.debug(msg=message, exc_info=e)
 
         return e
@@ -153,6 +200,16 @@ def get_reports(self, workspace: Workspace) -> List[Report]:
         reports: List[Report] = []
         try:
             reports = self._get_resolver().get_reports(workspace)
+            # Fill Report dataset
+            for report in reports:
+                if report.dataset_id:
+                    report.dataset = self.dataset_registry.get(report.dataset_id)
+                    if report.dataset is None:
+                        self.reporter.info(
+                            title="Missing Lineage For Report",
+                            message="A cross-workspace reference that failed to be resolved. Please ensure that no global workspace is being filtered out due to the workspace_id_pattern.",
+                            context=f"report-name: {report.name} and dataset-id: {report.dataset_id}",
+                        )
         except:
             self.log_http_error(
                 message=f"Unable to fetch reports for workspace {workspace.name}"
@@ -182,25 +239,36 @@ def fill_tags() -> None:
 
         fill_ownership()
         fill_tags()
-
         return reports
 
     def get_workspaces(self) -> List[Workspace]:
+        modified_workspace_ids: List[str] = []
+
         if self.__config.modified_since:
-            workspaces = self.get_modified_workspaces()
-            return workspaces
+            modified_workspace_ids = self.get_modified_workspaces()
 
         groups: List[dict] = []
+        filter_: Dict[str, str] = {}
         try:
-            groups = self._get_resolver().get_groups()
+            if modified_workspace_ids:
+                id_filter: List[str] = []
+
+                for id_ in modified_workspace_ids:
+                    id_filter.append(f"id eq {id_}")
+
+                filter_["$filter"] = " or ".join(id_filter)
+
+            groups = self._get_resolver().get_groups(filter_=filter_)
+
         except:
             self.log_http_error(message="Unable to fetch list of workspaces")
-            raise  # we want this exception to bubble up
+            # raise  # we want this exception to bubble up
 
         workspaces = [
             Workspace(
                 id=workspace[Constant.ID],
                 name=workspace[Constant.NAME],
+                type=workspace[Constant.TYPE],
                 datasets={},
                 dashboards=[],
                 reports=[],
@@ -208,39 +276,26 @@ def get_workspaces(self) -> List[Workspace]:
                 dashboard_endorsements={},
                 scan_result={},
                 independent_datasets=[],
+                app=None,  # It will be populated in _fill_metadata_from_scan_result method
             )
             for workspace in groups
         ]
         return workspaces
 
-    def get_modified_workspaces(self) -> List[Workspace]:
-        workspaces: List[Workspace] = []
+    def get_modified_workspaces(self) -> List[str]:
+        modified_workspace_ids: List[str] = []
 
         if self.__config.modified_since is None:
-            return workspaces
+            return modified_workspace_ids
 
         try:
             modified_workspace_ids = self.__admin_api_resolver.get_modified_workspaces(
                 self.__config.modified_since
             )
-            workspaces = [
-                Workspace(
-                    id=workspace_id,
-                    name="",
-                    datasets={},
-                    dashboards=[],
-                    reports=[],
-                    report_endorsements={},
-                    dashboard_endorsements={},
-                    scan_result={},
-                    independent_datasets=[],
-                )
-                for workspace_id in modified_workspace_ids
-            ]
         except:
             self.log_http_error(message="Unable to fetch list of modified workspaces.")
 
-        return workspaces
+        return modified_workspace_ids
 
     def _get_scan_result(self, workspace_ids: List[str]) -> Any:
         scan_id: Optional[str] = None
@@ -257,7 +312,7 @@ def _get_scan_result(self, workspace_ids: List[str]) -> Any:
                 )
             return None
 
-        logger.info("Waiting for scan to complete")
+        logger.debug("Waiting for scan to complete")
         if (
             self.__admin_api_resolver.wait_for_scan_to_complete(
                 scan_id=scan_id, timeout=self.__config.scan_timeout
@@ -309,22 +364,32 @@ def _get_workspace_datasets(self, workspace: Workspace) -> dict:
         logger.debug("Processing scan result for datasets")
 
         for dataset_dict in datasets:
-            dataset_instance: PowerBIDataset = self._get_resolver().get_dataset(
-                workspace_id=scan_result[Constant.ID],
-                dataset_id=dataset_dict[Constant.ID],
-            )
+            dataset_id = dataset_dict[Constant.ID]
+            try:
+                dataset_instance = self._get_resolver().get_dataset(
+                    workspace=workspace,
+                    dataset_id=dataset_id,
+                )
+                if dataset_instance is None:
+                    continue
+            except Exception as e:
+                self.reporter.warning(
+                    title="Unable to fetch dataset details",
+                    message="Skipping this dataset due to the error. Metadata will be incomplete.",
+                    context=f"workspace={workspace.name}, dataset-id={dataset_id}",
+                    exc=e,
+                )
+                continue
 
             # fetch + set dataset parameters
             try:
                 dataset_parameters = self._get_resolver().get_dataset_parameters(
-                    workspace_id=scan_result[Constant.ID],
-                    dataset_id=dataset_dict[Constant.ID],
+                    workspace_id=workspace.id,
+                    dataset_id=dataset_id,
                 )
                 dataset_instance.parameters = dataset_parameters
             except Exception as e:
-                logger.info(
-                    f"Unable to fetch dataset parameters for {dataset_dict[Constant.ID]}: {e}"
-                )
+                logger.info(f"Unable to fetch dataset parameters for {dataset_id}: {e}")
 
             if self.__config.extract_endorsements_to_tags:
                 dataset_instance.tags = self._parse_endorsement(
@@ -348,9 +413,11 @@ def _get_workspace_datasets(self, workspace: Workspace) -> dict:
                 )
                 table = Table(
                     name=table[Constant.NAME],
-                    full_name="{}.{}".format(
-                        dataset_name.replace(" ", "_"),
-                        table[Constant.NAME].replace(" ", "_"),
+                    full_name=form_full_table_name(
+                        config=self.__config,
+                        workspace=workspace,
+                        dataset_name=dataset_name,
+                        table_name=table[Constant.NAME],
                     ),
                     expression=expression,
                     columns=[
@@ -379,8 +446,90 @@ def _get_workspace_datasets(self, workspace: Workspace) -> dict:
                 dataset_instance.tables.append(table)
         return dataset_map
 
+    def get_app(
+        self,
+        app_id: str,
+    ) -> Optional[App]:
+        return self.__admin_api_resolver.get_app(
+            app_id=app_id,
+        )
+
+    def _populate_app_details(
+        self, workspace: Workspace, workspace_metadata: Dict
+    ) -> None:
+        # App_id is not present at the root level of workspace_metadata.
+        # It can be found in the workspace_metadata.dashboards or workspace_metadata.reports lists.
+
+        # Workspace_metadata contains duplicate entries for all dashboards and reports that we have included
+        # in the app.
+        # The duplicate entries for a report contain key `originalReportObjectId` referencing to
+        # an actual report id of workspace. The duplicate entries for a dashboard contain `displayName` where
+        # displayName is generated from displayName of original dashboard with prefix "App"
+        app_id: Optional[str] = None
+        app_reports: List[AppReport] = []
+        # Filter app reports
+        for report in workspace_metadata.get(Constant.REPORTS) or []:
+            if report.get(Constant.APP_ID):
+                app_reports.append(
+                    AppReport(
+                        id=report[Constant.ID],
+                        original_report_id=report[Constant.ORIGINAL_REPORT_OBJECT_ID],
+                    )
+                )
+                if app_id is None:  # In PowerBI one workspace can have one app
+                    app_id = report.get(Constant.APP_ID)
+
+        raw_app_dashboards: List[Dict] = []
+        # Filter app dashboards
+        for dashboard in workspace_metadata.get(Constant.DASHBOARDS) or []:
+            if dashboard.get(Constant.APP_ID):
+                raw_app_dashboards.append(dashboard)
+                if app_id is None:  # In PowerBI, one workspace contains one app
+                    app_id = report[Constant.APP_ID]
+
+        # workspace doesn't have an App. Above two loops can be avoided
+        # if app_id is available at root level in workspace_metadata
+        if app_id is None:
+            logger.debug(f"Workspace {workspace.name} does not contain an app.")
+            return
+
+        app: Optional[App] = self.get_app(app_id=app_id)
+        if app is None:
+            self.__reporter.info(
+                title="App Not Found",
+                message="The workspace includes an app, but its metadata is missing from the API response.",
+                context=f"workspace_name={workspace.name}",
+            )
+            return
+
+        # Map to find out which dashboards belongs to the App
+        workspace_dashboard_map: Dict[str, Dict] = {
+            raw_dashboard[Constant.DISPLAY_NAME]: raw_dashboard
+            for raw_dashboard in raw_app_dashboards
+        }
+
+        app_dashboards: List[AppDashboard] = []
+        for dashboard in workspace_metadata.get(Constant.DASHBOARDS) or []:
+            app_dashboard_display_name = f"[App] {dashboard[Constant.DISPLAY_NAME]}"  # A Dashboard is considered part of an App if the workspace_metadata contains a Dashboard with a label formatted as "[App] <DashboardName>".
+            if (
+                app_dashboard_display_name in workspace_dashboard_map
+            ):  # This dashboard is part of the App
+                app_dashboards.append(
+                    AppDashboard(
+                        id=workspace_dashboard_map[app_dashboard_display_name][
+                            Constant.ID
+                        ],
+                        original_dashboard_id=dashboard[Constant.ID],
+                    )
+                )
+
+        app.reports = app_reports
+        app.dashboards = app_dashboards
+        workspace.app = app
+
     def _fill_metadata_from_scan_result(
-        self, workspaces: List[Workspace]
+        self,
+        workspaces: List[Workspace],
     ) -> List[Workspace]:
         workspace_ids = [workspace.id for workspace in workspaces]
         scan_result = self._get_scan_result(workspace_ids)
@@ -389,9 +538,28 @@ def _fill_metadata_from_scan_result(
 
         workspaces = []
         for workspace_metadata in scan_result["workspaces"]:
+            if (
+                workspace_metadata.get(Constant.STATE) != Constant.ACTIVE
+                or workspace_metadata.get(Constant.TYPE)
+                not in self.__config.workspace_type_filter
+            ):
+                # if the state is not "Active" then in some state like Not Found, "name" attribute is not present
+                wrk_identifier: str = (
+                    workspace_metadata[Constant.NAME]
+                    if workspace_metadata.get(Constant.NAME)
+                    else workspace_metadata.get(Constant.ID)
+                )
+                self.__reporter.info(
+                    title="Skipped Workspace",
+                    message="Workspace was skipped due to the workspace_type_filter",
+                    context=f"workspace={wrk_identifier}",
+                )
+                continue
+
             cur_workspace = Workspace(
-                id=workspace_metadata["id"],
-                name=workspace_metadata["name"],
+                id=workspace_metadata[Constant.ID],
+                name=workspace_metadata[Constant.NAME],
+                type=workspace_metadata[Constant.TYPE],
                 datasets={},
                 dashboards=[],
                 reports=[],
@@ -399,11 +567,13 @@ def _fill_metadata_from_scan_result(
                 dashboard_endorsements={},
                 scan_result={},
                 independent_datasets=[],
+                app=None,  # It is getting set from scan-result
             )
             cur_workspace.scan_result = workspace_metadata
             cur_workspace.datasets = self._get_workspace_datasets(cur_workspace)
-
-            # Fetch endorsements tag if it is enabled from configuration
+            # collect all datasets in the registry
+            self.dataset_registry.update(cur_workspace.datasets)
+            # Fetch endorsement tag if it is enabled from configuration
             if self.__config.extract_endorsements_to_tags:
                 cur_workspace.dashboard_endorsements = self._get_dashboard_endorsements(
                     cur_workspace.scan_result
@@ -413,16 +583,18 @@ def _fill_metadata_from_scan_result(
                 )
             else:
                 logger.info(
-                    "Skipping endorsements tag as extract_endorsements_to_tags is set to "
-                    "false "
+                    "Skipping endorsements tag as extract_endorsements_to_tags is not enabled"
                 )
 
+            self._populate_app_details(
+                workspace=cur_workspace,
+                workspace_metadata=workspace_metadata,
+            )
             workspaces.append(cur_workspace)
 
         return workspaces
 
     def _fill_independent_datasets(self, workspace: Workspace) -> None:
-
         reachable_datasets: List[str] = []
         # Find out reachable datasets
         for dashboard in workspace.dashboards:
@@ -447,6 +619,16 @@ def fill_dashboards() -> None:
                 dashboard.tiles = self._get_resolver().get_tiles(
                     workspace, dashboard=dashboard
                 )
+                # set the dataset for tiles
+                for tile in dashboard.tiles:
+                    if tile.dataset_id:
+                        tile.dataset = self.dataset_registry.get(tile.dataset_id)
+                        if tile.dataset is None:
+                            self.reporter.info(
+                                title="Missing Lineage For Tile",
+                                message="A cross-workspace reference that failed to be resolved. Please ensure that no global workspace is being filtered out due to the workspace_id_pattern.",
+                                context=f"workspace-name: {workspace.name}, tile-name: {tile.title}, dataset-id: {tile.dataset_id}",
+                            )
 
         def fill_reports() -> None:
             if self.__config.extract_reports is False:
@@ -476,6 +658,10 @@ def fill_dashboard_tags() -> None:
     def fill_workspaces(
         self, workspaces: List[Workspace], reporter: PowerBiDashboardSourceReport
     ) -> Iterable[Workspace]:
+        logger.info(
+            f"Fetching initial metadata for workspaces: {[workspace.format_name_for_logger() for workspace in workspaces]}"
+        )
+
         workspaces = self._fill_metadata_from_scan_result(workspaces=workspaces)
         # First try to fill the admin detail as some regular metadata contains lineage to admin metadata
         for workspace in workspaces:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py
index 8854f9ff48348..2a247d0c63957 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py
@@ -126,7 +126,6 @@ def log_http_error(e: BaseException, message: str) -> Any:
 
 
 def get_response_dict(response: requests.Response, error_message: str) -> dict:
-
     result_dict: dict = {}
     try:
         response.raise_for_status()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/preset.py b/metadata-ingestion/src/datahub/ingestion/source/preset.py
index e51520898103d..7b0bc89648c52 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/preset.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/preset.py
@@ -56,7 +56,7 @@ class PresetConfig(SupersetConfig):
     def remove_trailing_slash(cls, v):
         return config_clean.remove_trailing_slashes(v)
 
-    @root_validator
+    @root_validator(skip_on_failure=True)
     def default_display_uri_to_connect_uri(cls, values):
         base = values.get("display_uri")
         if base is None:
@@ -85,6 +85,7 @@ def __init__(self, ctx: PipelineContext, config: PresetConfig):
         super().__init__(ctx, config)
         self.config = config
         self.report = StaleEntityRemovalSourceReport()
+        self.platform = "preset"
 
     def login(self):
         try:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py
index 790c1f918cdfd..f71949b9eb27f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py
@@ -78,8 +78,27 @@ class PulsarSchema:
     def __init__(self, schema):
         self.schema_version = schema.get("version")
 
-        avro_schema = json.loads(schema.get("data"))
-        self.schema_name = avro_schema.get("namespace") + "." + avro_schema.get("name")
+        schema_data = schema.get("data")
+        if not schema_data:
+            logger.warning("Schema data is empty or None. Using default empty schema.")
+            schema_data = "{}"
+
+        try:
+            avro_schema = json.loads(schema_data)
+        except json.JSONDecodeError as e:
+            logger.error(f"Invalid JSON schema: {schema_data}. Error: {str(e)}")
+            avro_schema = {}
+
+        self.schema_name = "null"
+        if avro_schema.get("namespace") and avro_schema.get("name"):
+            self.schema_name = (
+                avro_schema.get("namespace") + "." + avro_schema.get("name")
+            )
+        elif avro_schema.get("namespace"):
+            self.schema_name = avro_schema.get("namespace")
+        elif avro_schema.get("name"):
+            self.schema_name = avro_schema.get("name")
+
         self.schema_description = avro_schema.get("doc")
         self.schema_type = schema.get("type")
         self.schema_str = schema.get("data")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py
index 672fcbceb0603..a43f5f32493f2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py
@@ -15,6 +15,7 @@
     TimeType,
 )
 
+# TODO: Replace with standardized types in sql_types.py
 FIELD_TYPE_MAPPING: Dict[
     str,
     Type[
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py
index 38cf0bebcbc12..f11d1944029eb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redash.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redash.py
@@ -2,8 +2,7 @@
 import math
 import sys
 from dataclasses import dataclass, field
-from multiprocessing.pool import ThreadPool
-from typing import Dict, Iterable, List, Optional, Set, Type
+from typing import Dict, Iterable, List, Optional, Set
 
 import dateutil.parser as dp
 from packaging import version
@@ -23,7 +22,6 @@
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.registry import import_path
 from datahub.ingestion.api.source import Source, SourceCapability, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.com.linkedin.pegasus2avro.common import (
@@ -40,9 +38,10 @@
     ChartTypeClass,
     DashboardInfoClass,
 )
+from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
 from datahub.utilities.lossy_collections import LossyDict, LossyList
 from datahub.utilities.perf_timer import PerfTimer
-from datahub.utilities.sql_parser import SQLParser
+from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -270,10 +269,6 @@ class RedashConfig(ConfigModel):
     parse_table_names_from_sql: bool = Field(
         default=False, description="See note below."
     )
-    sql_parser: str = Field(
-        default="datahub.utilities.sql_parser.DefaultSQLParser",
-        description="custom SQL parser. See note below for details.",
-    )
 
     env: str = Field(
         default=DEFAULT_ENV,
@@ -354,7 +349,6 @@ def __init__(self, ctx: PipelineContext, config: RedashConfig):
         self.api_page_limit = self.config.api_page_limit or math.inf
 
         self.parse_table_names_from_sql = self.config.parse_table_names_from_sql
-        self.sql_parser_path = self.config.sql_parser
 
         logger.info(
             f"Running Redash ingestion with parse_table_names_from_sql={self.parse_table_names_from_sql}"
@@ -380,31 +374,6 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source:
         config = RedashConfig.parse_obj(config_dict)
         return cls(ctx, config)
 
-    @classmethod
-    def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]:
-        assert "." in sql_parser_path, "sql_parser-path must contain a ."
-        parser_cls = import_path(sql_parser_path)
-
-        if not issubclass(parser_cls, SQLParser):
-            raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}")
-        return parser_cls
-
-    @classmethod
-    def _get_sql_table_names(cls, sql: str, sql_parser_path: str) -> List[str]:
-        parser_cls = cls._import_sql_parser_cls(sql_parser_path)
-
-        try:
-            sql_table_names: List[str] = parser_cls(sql).get_tables()
-        except Exception as e:
-            logger.warning(f"Sql parser failed on {sql} with {e}")
-            return []
-
-        # Remove quotes from table names
-        sql_table_names = [t.replace('"', "") for t in sql_table_names]
-        sql_table_names = [t.replace("`", "") for t in sql_table_names]
-
-        return sql_table_names
-
     def _get_chart_data_source(self, data_source_id: Optional[int] = None) -> Dict:
         url = f"/api/data_sources/{data_source_id}"
         resp = self.client._get(url).json()
@@ -441,14 +410,6 @@ def _get_database_name_based_on_datasource(
 
         return database_name
 
-    def _construct_datalineage_urn(
-        self, platform: str, database_name: str, sql_table_name: str
-    ) -> str:
-        full_dataset_name = get_full_qualified_name(
-            platform, database_name, sql_table_name
-        )
-        return builder.make_dataset_urn(platform, full_dataset_name, self.config.env)
-
     def _get_datasource_urns(
         self, data_source: Dict, sql_query_data: Dict = {}
     ) -> Optional[List[str]]:
@@ -464,34 +425,23 @@ def _get_datasource_urns(
             # Getting table lineage from SQL parsing
             if self.parse_table_names_from_sql and data_source_syntax == "sql":
                 dataset_urns = list()
-                try:
-                    sql_table_names = self._get_sql_table_names(
-                        query, self.sql_parser_path
-                    )
-                except Exception as e:
+                sql_parser_in_tables = create_lineage_sql_parsed_result(
+                    query=query,
+                    platform=platform,
+                    env=self.config.env,
+                    platform_instance=None,
+                    default_db=database_name,
+                )
+                # make sure dataset_urns is not empty list
+                dataset_urns = sql_parser_in_tables.in_tables
+                if sql_parser_in_tables.debug_info.table_error:
                     self.report.queries_problem_parsing.add(str(query_id))
                     self.error(
                         logger,
                         "sql-parsing",
-                        f"exception {e} in parsing query-{query_id}-datasource-{data_source_id}",
+                        f"exception {sql_parser_in_tables.debug_info.table_error} in parsing query-{query_id}-datasource-{data_source_id}",
                     )
-                    sql_table_names = []
-                for sql_table_name in sql_table_names:
-                    try:
-                        dataset_urns.append(
-                            self._construct_datalineage_urn(
-                                platform, database_name, sql_table_name
-                            )
-                        )
-                    except Exception:
-                        self.report.queries_problem_parsing.add(str(query_id))
-                        self.warn(
-                            logger,
-                            "data-urn-invalid",
-                            f"Problem making URN for {sql_table_name} parsed from query {query_id}",
-                        )
 
-                # make sure dataset_urns is not empty list
                 return dataset_urns if len(dataset_urns) > 0 else None
 
             else:
@@ -646,11 +596,11 @@ def _emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]:
         self.report.total_dashboards = total_dashboards
         self.report.max_page_dashboards = max_page
 
-        dash_exec_pool = ThreadPool(self.config.parallelism)
-        for response in dash_exec_pool.imap_unordered(
-            self._process_dashboard_response, range(1, max_page + 1)
-        ):
-            yield from response
+        yield from ThreadedIteratorExecutor.process(
+            self._process_dashboard_response,
+            [(page,) for page in range(1, max_page + 1)],
+            max_workers=self.config.parallelism,
+        )
 
     def _get_chart_type_from_viz_data(self, viz_data: Dict) -> str:
         """
@@ -769,11 +719,12 @@ def _emit_chart_mces(self) -> Iterable[MetadataWorkUnit]:
         logger.info(f"/api/queries total count {total_queries} and max page {max_page}")
         self.report.total_queries = total_queries
         self.report.max_page_queries = max_page
-        chart_exec_pool = ThreadPool(self.config.parallelism)
-        for response in chart_exec_pool.imap_unordered(
-            self._process_query_response, range(1, max_page + 1)
-        ):
-            yield from response
+
+        yield from ThreadedIteratorExecutor.process(
+            self._process_query_response,
+            [(page,) for page in range(1, max_page + 1)],
+            max_workers=self.config.parallelism,
+        )
 
     def add_config_to_report(self) -> None:
         self.report.api_page_limit = self.config.api_page_limit
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
index 2ff73323a14e3..cad48eaf1c237 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
@@ -159,6 +159,7 @@ class RedshiftConfig(
         description="Whether to extract column level lineage. This config works with rest-sink only.",
     )
 
+    # TODO - use DatasetPropertiesConfigMixin instead
     patch_custom_properties: bool = Field(
         default=True,
         description="Whether to patch custom properties on existing datasets rather than replace.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py
index 43ad5bfcefdf1..ed0856fc1e292 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py
@@ -40,25 +40,25 @@ def report_redshift_failure(
     error_message = str(e).lower()
     if "permission denied" in error_message:
         if "svv_table_info" in error_message:
-            report.report_failure(
+            report.failure(
                 title="Permission denied",
                 message="Failed to extract metadata due to insufficient permission to access 'svv_table_info' table. Please ensure the provided database user has access.",
                 exc=e,
             )
         elif "svl_user_info" in error_message:
-            report.report_failure(
+            report.failure(
                 title="Permission denied",
                 message="Failed to extract metadata due to insufficient permission to access 'svl_user_info' table. Please ensure the provided database user has access.",
                 exc=e,
             )
         else:
-            report.report_failure(
+            report.failure(
                 title="Permission denied",
                 message="Failed to extract metadata due to insufficient permissions.",
                 exc=e,
             )
     else:
-        report.report_failure(
+        report.failure(
             title="Failed to extract some metadata",
             message="Failed to extract some metadata from Redshift.",
             exc=e,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
index 0e4cb6f1599e5..4b3d238a13261 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
@@ -53,6 +53,7 @@
 )
 from datahub.metadata.urns import DatasetUrn
 from datahub.sql_parsing.schema_resolver import SchemaResolver
+from datahub.sql_parsing.sql_parsing_aggregator import TableRename
 from datahub.sql_parsing.sqlglot_utils import get_dialect, parse_statement
 from datahub.utilities import memory_footprint
 from datahub.utilities.dedup_list import deduplicate_list
@@ -133,7 +134,7 @@ def parse_alter_table_rename(default_schema: str, query: str) -> Tuple[str, str,
     assert isinstance(parsed_query, sqlglot.exp.Alter)
     prev_name = parsed_query.this.name
     rename_clause = parsed_query.args["actions"][0]
-    assert isinstance(rename_clause, sqlglot.exp.RenameTable)
+    assert isinstance(rename_clause, sqlglot.exp.AlterRename)
     new_name = rename_clause.this.name
 
     schema = parsed_query.this.db or default_schema
@@ -504,21 +505,21 @@ def _populate_lineage_map(
             self.report_status(f"extract-{lineage_type.name}", False)
 
     def _update_lineage_map_for_table_renames(
-        self, table_renames: Dict[str, str]
+        self, table_renames: Dict[str, TableRename]
     ) -> None:
         if not table_renames:
             return
 
         logger.info(f"Updating lineage map for {len(table_renames)} table renames")
-        for new_table_urn, prev_table_urn in table_renames.items():
+        for entry in table_renames.values():
             # This table was renamed from some other name, copy in the lineage
             # for the previous name as well.
-            prev_table_lineage = self._lineage_map.get(prev_table_urn)
+            prev_table_lineage = self._lineage_map.get(entry.original_urn)
             if prev_table_lineage:
                 logger.debug(
-                    f"including lineage for {prev_table_urn} in {new_table_urn} due to table rename"
+                    f"including lineage for {entry.original_urn} in {entry.new_urn} due to table rename"
                 )
-                self._lineage_map[new_table_urn].merge_lineage(
+                self._lineage_map[entry.new_urn].merge_lineage(
                     upstreams=prev_table_lineage.upstreams,
                     cll=prev_table_lineage.cll,
                 )
@@ -672,7 +673,7 @@ def populate_lineage(
             for db, schemas in all_tables.items()
         }
 
-        table_renames: Dict[str, str] = {}
+        table_renames: Dict[str, TableRename] = {}
         if self.config.include_table_rename_lineage:
             table_renames, all_tables_set = self._process_table_renames(
                 database=database,
@@ -851,11 +852,11 @@ def _process_table_renames(
         database: str,
         connection: redshift_connector.Connection,
         all_tables: Dict[str, Dict[str, Set[str]]],
-    ) -> Tuple[Dict[str, str], Dict[str, Dict[str, Set[str]]]]:
+    ) -> Tuple[Dict[str, TableRename], Dict[str, Dict[str, Set[str]]]]:
         logger.info(f"Processing table renames for db {database}")
 
         # new urn -> prev urn
-        table_renames: Dict[str, str] = {}
+        table_renames: Dict[str, TableRename] = {}
 
         query = self.queries.alter_table_rename_query(
             db_name=database,
@@ -893,7 +894,9 @@ def _process_table_renames(
                 env=self.config.env,
             )
 
-            table_renames[new_urn] = prev_urn
+            table_renames[new_urn] = TableRename(
+                prev_urn, new_urn, query_text, timestamp=rename_row.start_time
+            )
 
             # We want to generate lineage for the previous name too.
             all_tables[database][schema].add(prev_name)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py
index 4b7f710beed08..4b84c25965a99 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py
@@ -5,6 +5,7 @@
 import redshift_connector
 
 from datahub.emitter import mce_builder
+from datahub.ingestion.api.closeable import Closeable
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.redshift.config import LineageMode, RedshiftConfig
@@ -39,7 +40,7 @@
 logger = logging.getLogger(__name__)
 
 
-class RedshiftSqlLineageV2:
+class RedshiftSqlLineageV2(Closeable):
     # does lineage and usage based on SQL parsing.
 
     def __init__(
@@ -56,6 +57,8 @@ def __init__(
         self.context = context
 
         self.database = database
+        self.known_urns: Set[str] = set()  # will be set later
+
         self.aggregator = SqlParsingAggregator(
             platform=self.platform,
             platform_instance=self.config.platform_instance,
@@ -66,6 +69,7 @@ def __init__(
             generate_operations=False,
             usage_config=self.config,
             graph=self.context.graph,
+            is_temp_table=self._is_temp_table,
         )
         self.report.sql_aggregator = self.aggregator.report
 
@@ -85,7 +89,16 @@ def __init__(
             self.report.lineage_end_time,
         ) = self._lineage_v1.get_time_window()
 
-        self.known_urns: Set[str] = set()  # will be set later
+    def _is_temp_table(self, name: str) -> bool:
+        return (
+            DatasetUrn.create_from_ids(
+                self.platform,
+                name,
+                env=self.config.env,
+                platform_instance=self.config.platform_instance,
+            ).urn()
+            not in self.known_urns
+        )
 
     def build(
         self,
@@ -105,15 +118,6 @@ def build(
             for schema, tables in schemas.items()
             for table in tables
         }
-        self.aggregator._is_temp_table = (
-            lambda name: DatasetUrn.create_from_ids(
-                self.platform,
-                name,
-                env=self.config.env,
-                platform_instance=self.config.platform_instance,
-            ).urn()
-            not in self.known_urns
-        )
 
         # Handle all the temp tables up front.
         if self.config.resolve_temp_table_in_lineage:
@@ -144,10 +148,8 @@ def build(
                     lambda: collections.defaultdict(set)
                 ),
             )
-            for new_urn, original_urn in table_renames.items():
-                self.aggregator.add_table_rename(
-                    original_urn=original_urn, new_urn=new_urn
-                )
+            for entry in table_renames.values():
+                self.aggregator.add_table_rename(entry)
 
         if self.config.table_lineage_mode in {
             LineageMode.SQL_BASED,
@@ -334,19 +336,26 @@ def _process_view_lineage(self, lineage_row: LineageRow) -> None:
         )
 
     def _process_copy_command(self, lineage_row: LineageRow) -> None:
-        source = self._lineage_v1._get_sources(
+        logger.debug(f"Processing COPY command for lineage row: {lineage_row}")
+        sources = self._lineage_v1._get_sources(
             lineage_type=LineageCollectorType.COPY,
             db_name=self.database,
             source_schema=None,
             source_table=None,
             ddl=None,
             filename=lineage_row.filename,
-        )[0]
+        )
+        logger.debug(f"Recognized sources: {sources}")
+        source = sources[0]
         if not source:
+            logger.debug("Ignoring command since couldn't recognize proper source")
             return
         s3_urn = source[0].urn
-
+        logger.debug(f"Recognized s3 dataset urn: {s3_urn}")
         if not lineage_row.target_schema or not lineage_row.target_table:
+            logger.debug(
+                f"Didn't find target schema (found: {lineage_row.target_schema}) or target table (found: {lineage_row.target_table})"
+            )
             return
         target = self._make_filtered_target(lineage_row)
         if not target:
@@ -429,3 +438,6 @@ def generate(self) -> Iterable[MetadataWorkUnit]:
                 message="Unexpected error(s) while attempting to extract lineage from SQL queries. See the full logs for more details.",
                 context=f"Query Parsing Failures: {self.aggregator.report.observed_query_parse_failures}",
             )
+
+    def close(self) -> None:
+        self.aggregator.close()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py
index affbcd00b5107..71a20890d35e8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py
@@ -4,6 +4,14 @@
 redshift_datetime_format = "%Y-%m-%d %H:%M:%S"
 
 
+# See https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext
+# for why we need to limit the size of the query text.
+# We use 290 instead instead of the standard 320, because escape characters can add to the length.
+_QUERY_SEQUENCE_LIMIT = 290
+
+_MAX_COPY_ENTRIES_PER_TABLE = 20
+
+
 class RedshiftCommonQuery:
     CREATE_TEMP_TABLE_CLAUSE = "create temp table"
     CREATE_TEMPORARY_TABLE_CLAUSE = "create temporary table"
@@ -283,6 +291,43 @@ def alter_table_rename_query(
             AND        SYS.query_text ILIKE '%alter table % rename to %'
         """
 
+    @staticmethod
+    def list_copy_commands_sql(
+        db_name: str, start_time: datetime, end_time: datetime
+    ) -> str:
+        return """\
+SELECT DISTINCT
+    target_schema,
+    target_table,
+    filename
+FROM (
+    SELECT
+        sti."schema" AS target_schema,
+        sti."table" AS target_table,
+        c.file_name AS filename,
+        ROW_NUMBER() OVER (
+            PARTITION BY sti."schema", sti."table"
+            ORDER BY si.start_time DESC
+        ) AS rn
+    FROM
+        SYS_QUERY_DETAIL AS si
+    JOIN SYS_LOAD_DETAIL AS c ON si.query_id = c.query_id
+    JOIN SVV_TABLE_INFO sti ON sti.table_id = si.table_id
+    WHERE
+        sti.database = '{db_name}'
+        AND si.start_time >= '{start_time}'
+        AND si.start_time < '{end_time}'
+) subquery
+WHERE rn <= {_MAX_COPY_ENTRIES_PER_TABLE}
+ORDER BY target_schema, target_table, filename
+""".format(
+            # We need the original database name for filtering
+            db_name=db_name,
+            start_time=start_time.strftime(redshift_datetime_format),
+            end_time=end_time.strftime(redshift_datetime_format),
+            _MAX_COPY_ENTRIES_PER_TABLE=_MAX_COPY_ENTRIES_PER_TABLE,
+        )
+
     @staticmethod
     def additional_table_metadata_query() -> str:
         raise NotImplementedError
@@ -317,12 +362,6 @@ def list_insert_create_queries_sql(
     ) -> str:
         raise NotImplementedError
 
-    @staticmethod
-    def list_copy_commands_sql(
-        db_name: str, start_time: datetime, end_time: datetime
-    ) -> str:
-        raise NotImplementedError
-
 
 class RedshiftProvisionedQuery(RedshiftCommonQuery):
     @staticmethod
@@ -465,99 +504,70 @@ def list_unload_commands_sql(
     def list_insert_create_queries_sql(
         db_name: str, start_time: datetime, end_time: datetime
     ) -> str:
-        return """
-            with query_txt as
-                (
-                    select
-                        query,
-                        pid,
-                        LISTAGG(case
-                            when LEN(RTRIM(text)) = 0 then text
-                            else RTRIM(text)
-                        end) within group (
-                    order by
-                        sequence) as ddl
-                    from
-                        (
-                        select
-                            query,
-                            pid,
-                            text,
-                            sequence
-                        from
-                            STL_QUERYTEXT
-                        where
-                            sequence < 320
-                        order by
-                            sequence
-                    )
-                    group by
-                        query,
-                        pid
-                )
-                        select
-                    distinct tbl as target_table_id,
-                    sti.schema as target_schema,
-                    sti.table as target_table,
-                    sti.database as cluster,
-                    usename as username,
-                    ddl,
-                    sq.query as query_id,
-                    min(si.starttime) as timestamp,
-                    ANY_VALUE(pid) as session_id
-                from
-                    stl_insert as si
-                left join SVV_TABLE_INFO sti on
-                    sti.table_id = tbl
-                left join svl_user_info sui on
-                    si.userid = sui.usesysid
-                left join query_txt sq on
-                    si.query = sq.query
-                left join stl_load_commits slc on
-                    slc.query = si.query
-                where
-                        sui.usename <> 'rdsdb'
-                        and cluster = '{db_name}'
-                        and slc.query IS NULL
-                        and si.starttime >= '{start_time}'
-                        and si.starttime < '{end_time}'
-                group by
-                    target_table_id,
-                    target_schema,
-                    target_table,
-                    cluster,
-                    username,
-                    ddl,
-                    sq.query
+        return """\
+with query_txt as (
+    select
+        query,
+        pid,
+        LISTAGG(case
+            when LEN(RTRIM(text)) = 0 then text
+            else RTRIM(text)
+        end) within group (
+            order by sequence
+        ) as ddl
+    from (
+        select
+            query,
+            pid,
+            text,
+            sequence
+        from
+            STL_QUERYTEXT
+        where
+            sequence < {_QUERY_SEQUENCE_LIMIT}
+        order by
+            sequence
+    )
+    group by
+        query,
+        pid
+)
+select
+    distinct tbl as target_table_id,
+    sti.schema as target_schema,
+    sti.table as target_table,
+    sti.database as cluster,
+    usename as username,
+    ddl,
+    sq.query as query_id,
+    min(si.starttime) as timestamp,
+    ANY_VALUE(pid) as session_id
+from
+    stl_insert as si
+left join SVV_TABLE_INFO sti on
+    sti.table_id = tbl
+left join svl_user_info sui on
+    si.userid = sui.usesysid
+left join query_txt sq on
+    si.query = sq.query
+left join stl_load_commits slc on
+    slc.query = si.query
+where
+        sui.usename <> 'rdsdb'
+        and cluster = '{db_name}'
+        and slc.query IS NULL
+        and si.starttime >= '{start_time}'
+        and si.starttime < '{end_time}'
+group by
+    target_table_id,
+    target_schema,
+    target_table,
+    cluster,
+    username,
+    ddl,
+    sq.query
         """.format(
-            # We need the original database name for filtering
-            db_name=db_name,
-            start_time=start_time.strftime(redshift_datetime_format),
-            end_time=end_time.strftime(redshift_datetime_format),
-        )
-
-    @staticmethod
-    def list_copy_commands_sql(
-        db_name: str, start_time: datetime, end_time: datetime
-    ) -> str:
-        return """
-                select
-                    distinct
-                        "schema" as target_schema,
-                        "table" as target_table,
-                        filename
-                from
-                    stl_insert as si
-                join stl_load_commits as c on
-                    si.query = c.query
-                join SVV_TABLE_INFO sti on
-                    sti.table_id = tbl
-                where
-                    database = '{db_name}'
-                    and si.starttime >= '{start_time}'
-                    and si.starttime < '{end_time}'
-                order by target_schema, target_table, starttime asc
-                """.format(
+            _QUERY_SEQUENCE_LIMIT=_QUERY_SEQUENCE_LIMIT,
             # We need the original database name for filtering
             db_name=db_name,
             start_time=start_time.strftime(redshift_datetime_format),
@@ -571,84 +581,81 @@ def temp_table_ddl_query(start_time: datetime, end_time: datetime) -> str:
         end_time_str: str = end_time.strftime(redshift_datetime_format)
 
         return rf"""-- DataHub Redshift Source temp table DDL query
+select
+    *
+from (
+    select
+        session_id,
+        transaction_id,
+        start_time,
+        userid,
+        REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\n','\\n'), '(CREATE(?:[\\n\\s\\t]+(?:temp|temporary))?(?:[\\n\\s\\t]+)table(?:[\\n\\s\\t]+)[^\\n\\s\\t()-]+)', 0, 1, 'ipe'),'[\\n\\s\\t]+',' ',1,'p') as create_command,
+        query_text,
+        row_number() over (
+            partition by session_id, TRIM(query_text)
+            order by start_time desc
+        ) rn
+    from (
+        select
+            pid as session_id,
+            xid as transaction_id,
+            starttime as start_time,
+            type,
+            query_text,
+            userid
+        from (
             select
-                *
+                starttime,
+                pid,
+                xid,
+                type,
+                userid,
+                LISTAGG(case
+                    when LEN(RTRIM(text)) = 0 then text
+                    else RTRIM(text)
+                end,
+                '') within group (
+                    order by sequence
+                ) as query_text
             from
-                (
-                select
-                    session_id,
-                    transaction_id,
-                    start_time,
-                    userid,
-                    REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\n','\\n'), '(CREATE(?:[\\n\\s\\t]+(?:temp|temporary))?(?:[\\n\\s\\t]+)table(?:[\\n\\s\\t]+)[^\\n\\s\\t()-]+)', 0, 1, 'ipe'),'[\\n\\s\\t]+',' ',1,'p') as create_command,
-                    query_text,
-                    row_number() over (
-                        partition by session_id, TRIM(query_text)
-                        order by start_time desc
-                    ) rn
-                from
-                    (
-                    select
-                        pid as session_id,
-                        xid as transaction_id,
-                        starttime as start_time,
-                        type,
-                        query_text,
-                        userid
-                    from
-                        (
-                        select
-                            starttime,
-                            pid,
-                            xid,
-                            type,
-                            userid,
-                            LISTAGG(case
-                                when LEN(RTRIM(text)) = 0 then text
-                                else RTRIM(text)
-                            end,
-                            '') within group (
-                                order by sequence
-                            ) as query_text
-                        from
-                            SVL_STATEMENTTEXT
-                        where
-                            type in ('DDL', 'QUERY')
-                            AND        starttime >= '{start_time_str}'
-                            AND        starttime < '{end_time_str}'
-                            -- See https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext
-                            AND sequence < 320
-                        group by
-                            starttime,
-                            pid,
-                            xid,
-                            type,
-                            userid
-                        order by
-                            starttime,
-                            pid,
-                            xid,
-                            type,
-                            userid
-                            asc)
-                    where
-                        type in ('DDL', 'QUERY')
-                )
-                where
-                    (create_command ilike 'create temp table %'
-                        or create_command ilike 'create temporary table %'
-                        -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction
-                        or create_command ilike 'create table %')
-                    -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out.
-                    and query_text not ilike 'CREATE TEMP TABLE volt_tt_%'
-                    and create_command not like 'CREATE TEMP TABLE volt_tt_'
-                    -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query
-                    and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%'
-
-            )
+                SVL_STATEMENTTEXT
             where
-                rn = 1
-            """
+                type in ('DDL', 'QUERY')
+                AND        starttime >= '{start_time_str}'
+                AND        starttime < '{end_time_str}'
+                AND sequence < {_QUERY_SEQUENCE_LIMIT}
+            group by
+                starttime,
+                pid,
+                xid,
+                type,
+                userid
+            order by
+                starttime,
+                pid,
+                xid,
+                type,
+                userid
+                asc
+        )
+        where
+            type in ('DDL', 'QUERY')
+    )
+    where
+        (create_command ilike 'create temp table %'
+            or create_command ilike 'create temporary table %'
+            -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction
+            or create_command ilike 'create table %')
+        -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out.
+        and query_text not ilike 'CREATE TEMP TABLE volt_tt_%'
+        and create_command not like 'CREATE TEMP TABLE volt_tt_'
+        -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query
+        and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%'
+
+)
+where
+    rn = 1
+"""
 
     # Add this join to the sql query for more metrics on completed queries
     # LEFT JOIN svl_query_metrics_summary sqms ON ss.query = sqms.query
@@ -941,33 +948,6 @@ def list_insert_create_queries_sql(
 
     # when loading from s3 using prefix with a single file it produces 2 lines (for file and just directory) - also
     # behaves like this when run in the old way
-    @staticmethod
-    def list_copy_commands_sql(
-        db_name: str, start_time: datetime, end_time: datetime
-    ) -> str:
-        return """
-                select
-                    distinct
-                        "schema" as target_schema,
-                        "table" as target_table,
-                        c.file_name
-                from
-                    SYS_QUERY_DETAIL as si
-                join SYS_LOAD_DETAIL as c on
-                    si.query_id = c.query_id
-                join SVV_TABLE_INFO sti on
-                    sti.table_id = si.table_id
-                where
-                    database = '{db_name}'
-                    and si.start_time >= '{start_time}'
-                    and si.start_time < '{end_time}'
-                order by target_schema, target_table, si.start_time asc
-                """.format(
-            # We need the original database name for filtering
-            db_name=db_name,
-            start_time=start_time.strftime(redshift_datetime_format),
-            end_time=end_time.strftime(redshift_datetime_format),
-        )
 
     # handles "create table IF ..." statements wrong probably - "create command" field contains only "create table if" in such cases
     # also similar happens if for example table name contains special characters quoted with " i.e. "test-table1"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
index a9fc9ab8f3e99..49f7941563c1a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
@@ -222,6 +222,7 @@ class RedshiftSource(StatefulIngestionSourceBase, TestableSource):
     ```
     """
 
+    # TODO: Replace with standardized types in sql_types.py
     REDSHIFT_FIELD_TYPE_MAPPINGS: Dict[
         str,
         Type[
@@ -436,7 +437,6 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit
     def _extract_metadata(
         self, connection: redshift_connector.Connection, database: str
     ) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
-
         yield from self.gen_database_container(
             database=database,
         )
@@ -451,24 +451,23 @@ def _extract_metadata(
         )
 
         if self.config.use_lineage_v2:
-            lineage_extractor = RedshiftSqlLineageV2(
+            with RedshiftSqlLineageV2(
                 config=self.config,
                 report=self.report,
                 context=self.ctx,
                 database=database,
                 redundant_run_skip_handler=self.redundant_lineage_run_skip_handler,
-            )
-
-            yield from lineage_extractor.aggregator.register_schemas_from_stream(
-                self.process_schemas(connection, database)
-            )
+            ) as lineage_extractor:
+                yield from lineage_extractor.aggregator.register_schemas_from_stream(
+                    self.process_schemas(connection, database)
+                )
 
-            self.report.report_ingestion_stage_start(LINEAGE_EXTRACTION)
-            yield from self.extract_lineage_v2(
-                connection=connection,
-                database=database,
-                lineage_extractor=lineage_extractor,
-            )
+                self.report.report_ingestion_stage_start(LINEAGE_EXTRACTION)
+                yield from self.extract_lineage_v2(
+                    connection=connection,
+                    database=database,
+                    lineage_extractor=lineage_extractor,
+                )
 
             all_tables = self.get_all_tables()
         else:
@@ -832,6 +831,8 @@ def gen_dataset_workunits(
             customProperties=custom_properties,
         )
         if self.config.patch_custom_properties:
+            # TODO: use auto_incremental_properties workunit processor instead
+            # Deprecate use of patch_custom_properties
             patch_builder = create_dataset_props_patch_builder(
                 dataset_urn, dataset_properties
             )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py
index ff28ed2c5e849..2748f2a588a93 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py
@@ -3,7 +3,7 @@
 from typing import Dict, Optional
 
 from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin
-from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.ingestion.source_report.time_window import BaseTimeWindowReport
 from datahub.sql_parsing.sql_parsing_aggregator import SqlAggregatorReport
@@ -14,7 +14,7 @@
 
 @dataclass
 class RedshiftReport(
-    ProfilingSqlReport,
+    SQLSourceReport,
     IngestionStageReport,
     BaseTimeWindowReport,
     ClassificationReportMixin,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
index 7ca89a56d36d9..ceac9e96d1ddd 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
@@ -9,6 +9,7 @@
 from itertools import groupby
 from pathlib import PurePath
 from typing import Any, Dict, Iterable, List, Optional, Tuple
+from urllib.parse import urlparse
 
 import smart_open.compression as so_compression
 from more_itertools import peekable
@@ -804,7 +805,6 @@ def get_dir_to_process(
         protocol: str,
         min: bool = False,
     ) -> List[str]:
-
         # if len(path_spec.include.split("/")) == len(f"{protocol}{bucket_name}/{folder}".split("/")):
         #    return [f"{protocol}{bucket_name}/{folder}"]
 
@@ -994,9 +994,7 @@ def s3_browser(self, path_spec: PathSpec, sample_size: int) -> Iterable[BrowsePa
                         folders = []
                         for dir in dirs_to_process:
                             logger.info(f"Getting files from folder: {dir}")
-                            prefix_to_process = dir.rstrip("\\").lstrip(
-                                self.create_s3_path(bucket_name, "/")
-                            )
+                            prefix_to_process = urlparse(dir).path.lstrip("/")
 
                             folders.extend(
                                 self.get_folder_info(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py
index 8309c469f67c5..b75f15c0ce770 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py
@@ -13,12 +13,9 @@
 from urllib3.util.retry import Retry
 
 from datahub.configuration.common import AllowDenyPattern
-from datahub.configuration.source_common import (
-    DEFAULT_ENV,
-    DatasetSourceConfigMixin,
-    EnvConfigMixin,
-)
+from datahub.configuration.source_common import DatasetSourceConfigMixin, EnvConfigMixin
 from datahub.emitter.mce_builder import (
+    DEFAULT_ENV,
     dataset_urn_to_key,
     make_dashboard_urn,
     make_data_platform_urn,
@@ -329,7 +326,9 @@ def get_resource_workunits(
             entityUrn=dashboard_urn,
             aspect=DashboardInfoClass(
                 title=resource.name,
-                description=resource.description,
+                description=resource.description
+                if resource.description is not None
+                else "",
                 lastModified=ChangeAuditStampsClass(
                     created=AuditStampClass(
                         time=round(resource.created_time.timestamp() * 1000),
@@ -402,7 +401,6 @@ def get_model_workunits(
 
             columns = self.get_import_data_model_columns(model_id=model.model_id)
             for column in columns:
-
                 schema_field = SchemaFieldClass(
                     fieldPath=column.name,
                     type=self.get_schema_field_data_type(column),
@@ -559,7 +557,14 @@ def get_sac_connection(
 
         retries = 3
         backoff_factor = 10
-        status_forcelist = (500,)
+
+        # The Resources and Data Import Service APIs of SAP Analytics Cloud can be somewhat unstable, occasionally
+        # returning HTTP errors for some requests, even though the APIs are generally operational. Therefore, we must
+        # retry these requests to increase the likelihood that the ingestion is successful. For the same reason we
+        # should also retry requests that receive a 401 HTTP status; however, this status also legitimately indicates
+        # that the provided OAuth credentials are invalid or that the OAuth client does not have the correct
+        # permissions assigned, therefore requests that receive a 401 HTTP status must not be retried.
+        status_forcelist = (400, 500, 503)
 
         retry = Retry(
             total=retries,
@@ -611,7 +616,9 @@ def get_resources(self) -> Iterable[Resource]:
         entity: pyodata.v2.service.EntityProxy
         for entity in entities:
             resource_id: str = entity.resourceId
-            name: str = entity.name.strip()
+            name: str = (
+                entity.name.strip() if entity.name is not None else entity.resourceId
+            )
 
             if not self.config.resource_id_pattern.allowed(
                 resource_id
@@ -655,8 +662,12 @@ def get_resources(self) -> Iterable[Resource]:
                     ResourceModel(
                         namespace=namespace,
                         model_id=model_id,
-                        name=nav_entity.name.strip(),
-                        description=nav_entity.description.strip(),
+                        name=nav_entity.name.strip()
+                        if nav_entity.name is not None
+                        else f"{namespace}:{model_id}",
+                        description=nav_entity.description.strip()
+                        if nav_entity.description is not None
+                        else None,
                         system_type=nav_entity.systemType,  # BW or HANA
                         connection_id=nav_entity.connectionId,
                         external_id=nav_entity.externalId,  # query:[][][query] or view:[schema][schema.namespace][view]
@@ -678,7 +689,9 @@ def get_resources(self) -> Iterable[Resource]:
                 resource_subtype=entity.resourceSubtype,
                 story_id=entity.storyId,
                 name=name,
-                description=entity.description.strip(),
+                description=entity.description.strip()
+                if entity.description is not None
+                else None,
                 created_time=entity.createdTime,
                 created_by=created_by,
                 modified_time=entity.modifiedTime,
@@ -715,7 +728,11 @@ def get_import_data_model_columns(
             columns.append(
                 ImportDataModelColumn(
                     name=column["columnName"].strip(),
-                    description=column["descriptionName"].strip(),
+                    description=(
+                        column["descriptionName"].strip()
+                        if column.get("descriptionName") is not None
+                        else None
+                    ),
                     property_type=column["propertyType"],
                     data_type=column["columnDataType"],
                     max_length=column.get("maxLength"),
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py
index 457fda1e06181..2c02b444cea1c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py
@@ -8,7 +8,7 @@ class ResourceModel:
     namespace: str
     model_id: str
     name: str
-    description: str
+    description: Optional[str]
     system_type: Optional[str]
     connection_id: Optional[str]
     external_id: Optional[str]
@@ -22,7 +22,7 @@ class Resource:
     resource_subtype: str
     story_id: str
     name: str
-    description: str
+    description: Optional[str]
     created_time: datetime
     created_by: Optional[str]
     modified_time: datetime
@@ -36,7 +36,7 @@ class Resource:
 @dataclass(frozen=True)
 class ImportDataModelColumn:
     name: str
-    description: str
+    description: Optional[str]
     property_type: str
     data_type: str
     max_length: Optional[int]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/csv_tsv.py b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/csv_tsv.py
index 54f7dfb5b903c..ab7b887cba1d8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/csv_tsv.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/csv_tsv.py
@@ -3,15 +3,15 @@
 from tableschema import Table
 
 from datahub.ingestion.source.schema_inference.base import SchemaInferenceBase
-from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     BooleanTypeClass,
     DateTypeClass,
     NullTypeClass,
     NumberTypeClass,
     RecordTypeClass,
-    SchemaField,
-    SchemaFieldDataType,
+    SchemaFieldClass as SchemaField,
+    SchemaFieldDataTypeClass as SchemaFieldDataType,
     StringTypeClass,
     TimeTypeClass,
     UnionTypeClass,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/json.py b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/json.py
index 1f2c73a2522d0..1659aaf6fa202 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/json.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/json.py
@@ -7,14 +7,14 @@
 
 from datahub.ingestion.source.schema_inference.base import SchemaInferenceBase
 from datahub.ingestion.source.schema_inference.object import construct_schema
-from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     BooleanTypeClass,
     NullTypeClass,
     NumberTypeClass,
     RecordTypeClass,
-    SchemaField,
-    SchemaFieldDataType,
+    SchemaFieldClass as SchemaField,
+    SchemaFieldDataTypeClass as SchemaFieldDataType,
     StringTypeClass,
     UnionTypeClass,
 )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/parquet.py b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/parquet.py
index 1f3f2e0a1e8a8..efc605e0df8ca 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/parquet.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/parquet.py
@@ -4,7 +4,7 @@
 import pyarrow.parquet
 
 from datahub.ingestion.source.schema_inference.base import SchemaInferenceBase
-from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     BooleanTypeClass,
     BytesTypeClass,
@@ -12,8 +12,8 @@
     NullTypeClass,
     NumberTypeClass,
     RecordTypeClass,
-    SchemaField,
-    SchemaFieldDataType,
+    SchemaFieldClass as SchemaField,
+    SchemaFieldDataTypeClass as SchemaFieldDataType,
     StringTypeClass,
     TimeTypeClass,
     UnionTypeClass,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py
index 922b0be3b4a93..5a657d804cb7b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py
@@ -80,6 +80,7 @@ class Workbook(BaseModel):
     path: str
     latestVersion: int
     workspaceId: Optional[str] = None
+    description: Optional[str] = None
     pages: List[Page] = []
     badge: Optional[str] = None
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py
index dd4b65a2cbdf2..e96eeb58d96ef 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py
@@ -4,7 +4,12 @@
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import ConfigurationError
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers
+from datahub.emitter.mcp_builder import (
+    add_entity_to_container,
+    add_owner_to_entity_wu,
+    add_tags_to_entity_wu,
+    gen_containers,
+)
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SourceCapability,
@@ -59,12 +64,14 @@
     UpstreamLineage,
 )
 from datahub.metadata.schema_classes import (
+    AuditStampClass,
     BrowsePathEntryClass,
     BrowsePathsV2Class,
     ChangeAuditStampsClass,
     ChartInfoClass,
     DashboardInfoClass,
     DataPlatformInstanceClass,
+    EdgeClass,
     GlobalTagsClass,
     InputFieldClass,
     InputFieldsClass,
@@ -74,6 +81,7 @@
     SchemaFieldClass,
     SchemaFieldDataTypeClass,
     StringTypeClass,
+    SubTypesClass,
     TagAssociationClass,
 )
 from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
@@ -257,11 +265,6 @@ def _gen_entity_browsepath_aspect(
         entries = [
             BrowsePathEntryClass(id=parent_entity_urn, urn=parent_entity_urn)
         ] + [BrowsePathEntryClass(id=path) for path in paths]
-        if self.config.platform_instance:
-            urn = builder.make_dataplatform_instance_urn(
-                self.platform, self.config.platform_instance
-            )
-            entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries
         return MetadataChangeProposalWrapper(
             entityUrn=entity_urn,
             aspect=BrowsePathsV2Class(entries),
@@ -424,11 +427,11 @@ def _gen_elements_workunit(
         elements: List[Element],
         workbook: Workbook,
         all_input_fields: List[InputFieldClass],
+        paths: List[str],
     ) -> Iterable[MetadataWorkUnit]:
         """
         Map Sigma page element to Datahub Chart
         """
-
         for element in elements:
             chart_urn = builder.make_chart_urn(
                 platform=self.platform,
@@ -459,11 +462,14 @@ def _gen_elements_workunit(
                 ),
             ).as_workunit()
 
-            yield from add_entity_to_container(
-                container_key=self._gen_workbook_key(workbook.workbookId),
-                entity_type="chart",
-                entity_urn=chart_urn,
-            )
+            if workbook.workspaceId:
+                yield self._gen_entity_browsepath_aspect(
+                    entity_urn=chart_urn,
+                    parent_entity_urn=builder.make_container_urn(
+                        self._gen_workspace_key(workbook.workspaceId)
+                    ),
+                    paths=paths + [workbook.name],
+                )
 
             # Add sigma dataset's upstream dataset urn mapping
             for dataset_urn, upstream_dataset_urns in inputs.items():
@@ -494,7 +500,9 @@ def _gen_elements_workunit(
 
             all_input_fields.extend(element_input_fields)
 
-    def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]:
+    def _gen_pages_workunit(
+        self, workbook: Workbook, paths: List[str]
+    ) -> Iterable[MetadataWorkUnit]:
         """
         Map Sigma workbook page to Datahub dashboard
         """
@@ -505,20 +513,23 @@ def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]:
 
             yield self._gen_dashboard_info_workunit(page)
 
-            yield from add_entity_to_container(
-                container_key=self._gen_workbook_key(workbook.workbookId),
-                entity_type="dashboard",
-                entity_urn=dashboard_urn,
-            )
-
             dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn)
             if dpi_aspect:
                 yield dpi_aspect
 
             all_input_fields: List[InputFieldClass] = []
 
+            if workbook.workspaceId:
+                yield self._gen_entity_browsepath_aspect(
+                    entity_urn=dashboard_urn,
+                    parent_entity_urn=builder.make_container_urn(
+                        self._gen_workspace_key(workbook.workspaceId)
+                    ),
+                    paths=paths + [workbook.name],
+                )
+
             yield from self._gen_elements_workunit(
-                page.elements, workbook, all_input_fields
+                page.elements, workbook, all_input_fields, paths
             )
 
             yield MetadataChangeProposalWrapper(
@@ -531,42 +542,89 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni
         Map Sigma Workbook to Datahub container
         """
         owner_username = self.sigma_api.get_user_name(workbook.createdBy)
-        workbook_key = self._gen_workbook_key(workbook.workbookId)
-        yield from gen_containers(
-            container_key=workbook_key,
-            name=workbook.name,
-            sub_types=[BIContainerSubTypes.SIGMA_WORKBOOK],
-            parent_container_key=(
-                self._gen_workspace_key(workbook.workspaceId)
-                if workbook.workspaceId
-                else None
+
+        dashboard_urn = self._gen_dashboard_urn(workbook.workbookId)
+
+        yield self._gen_entity_status_aspect(dashboard_urn)
+
+        lastModified = AuditStampClass(
+            time=int(workbook.updatedAt.timestamp() * 1000),
+            actor="urn:li:corpuser:datahub",
+        )
+        created = AuditStampClass(
+            time=int(workbook.createdAt.timestamp() * 1000),
+            actor="urn:li:corpuser:datahub",
+        )
+
+        dashboard_info_cls = DashboardInfoClass(
+            title=workbook.name,
+            description=workbook.description if workbook.description else "",
+            dashboards=[
+                EdgeClass(
+                    destinationUrn=self._gen_dashboard_urn(page.get_urn_part()),
+                    sourceUrn=dashboard_urn,
+                )
+                for page in workbook.pages
+            ],
+            externalUrl=workbook.url,
+            lastModified=ChangeAuditStampsClass(
+                created=created, lastModified=lastModified
             ),
-            extra_properties={
+            customProperties={
                 "path": workbook.path,
                 "latestVersion": str(workbook.latestVersion),
             },
-            owner_urn=(
-                builder.make_user_urn(owner_username)
-                if self.config.ingest_owner and owner_username
-                else None
-            ),
-            external_url=workbook.url,
-            tags=[workbook.badge] if workbook.badge else None,
-            created=int(workbook.createdAt.timestamp() * 1000),
-            last_modified=int(workbook.updatedAt.timestamp() * 1000),
         )
+        yield MetadataChangeProposalWrapper(
+            entityUrn=dashboard_urn, aspect=dashboard_info_cls
+        ).as_workunit()
+
+        # Set subtype
+        yield MetadataChangeProposalWrapper(
+            entityUrn=dashboard_urn,
+            aspect=SubTypesClass(typeNames=[BIContainerSubTypes.SIGMA_WORKBOOK]),
+        ).as_workunit()
+
+        # Ownership
+        owner_urn = (
+            builder.make_user_urn(owner_username)
+            if self.config.ingest_owner and owner_username
+            else None
+        )
+        if owner_urn:
+            yield from add_owner_to_entity_wu(
+                entity_type="dashboard",
+                entity_urn=dashboard_urn,
+                owner_urn=owner_urn,
+            )
+
+        # Tags
+        tags = [workbook.badge] if workbook.badge else None
+        if tags:
+            yield from add_tags_to_entity_wu(
+                entity_type="dashboard",
+                entity_urn=dashboard_urn,
+                tags=sorted(tags),
+            )
 
         paths = workbook.path.split("/")[1:]
-        if len(paths) > 0 and workbook.workspaceId:
+        if workbook.workspaceId:
             yield self._gen_entity_browsepath_aspect(
-                entity_urn=builder.make_container_urn(workbook_key),
+                entity_urn=dashboard_urn,
                 parent_entity_urn=builder.make_container_urn(
                     self._gen_workspace_key(workbook.workspaceId)
                 ),
-                paths=paths,
+                paths=paths + [workbook.name],
             )
 
-        yield from self._gen_pages_workunit(workbook)
+            if len(paths) == 0:
+                yield from add_entity_to_container(
+                    container_key=self._gen_workspace_key(workbook.workspaceId),
+                    entity_type="dashboard",
+                    entity_urn=dashboard_urn,
+                )
+
+        yield from self._gen_pages_workunit(workbook, paths)
 
     def _gen_sigma_dataset_upstream_lineage_workunit(
         self,
diff --git a/metadata-ingestion/src/datahub/configuration/oauth.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_config.py
similarity index 100%
rename from metadata-ingestion/src/datahub/configuration/oauth.py
rename to metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_config.py
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/oauth_generator.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_generator.py
similarity index 97%
rename from metadata-ingestion/src/datahub/ingestion/source/sql/oauth_generator.py
rename to metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_generator.py
index 7231c6ef6b1df..a2dc0118b3978 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/oauth_generator.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_generator.py
@@ -8,7 +8,7 @@
 from OpenSSL.crypto import FILETYPE_PEM, load_certificate
 from pydantic.types import SecretStr
 
-from datahub.configuration.oauth import OAuthIdentityProvider
+from datahub.ingestion.source.snowflake.oauth_config import OAuthIdentityProvider
 
 logger = logging.getLogger(__name__)
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py
index 229c0e292fbaf..1d1cc3c2af4f0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py
@@ -1,7 +1,7 @@
 import logging
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Dict, List, Optional, Set, cast
+from typing import Dict, List, Optional, Set
 
 import pydantic
 from pydantic import Field, SecretStr, root_validator, validator
@@ -16,6 +16,9 @@
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
 from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
+from datahub.ingestion.api.incremental_properties_helper import (
+    IncrementalPropertiesConfigMixin,
+)
 from datahub.ingestion.glossary.classification_mixin import (
     ClassificationSourceConfigMixin,
 )
@@ -118,9 +121,10 @@ def validate_legacy_schema_pattern(cls, values: Dict) -> Dict:
             )
 
         # Always exclude reporting metadata for INFORMATION_SCHEMA schema
-        if schema_pattern is not None and schema_pattern:
+        if schema_pattern:
             logger.debug("Adding deny for INFORMATION_SCHEMA to schema_pattern.")
-            cast(AllowDenyPattern, schema_pattern).deny.append(r".*INFORMATION_SCHEMA$")
+            assert isinstance(schema_pattern, AllowDenyPattern)
+            schema_pattern.deny.append(r".*INFORMATION_SCHEMA$")
 
         return values
 
@@ -187,6 +191,7 @@ class SnowflakeV2Config(
     StatefulUsageConfigMixin,
     StatefulProfilingConfigMixin,
     ClassificationSourceConfigMixin,
+    IncrementalPropertiesConfigMixin,
 ):
     include_usage_stats: bool = Field(
         default=True,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py
index d39e95a884dbc..2239338972d9b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py
@@ -16,13 +16,17 @@
 
 from datahub.configuration.common import ConfigModel, ConfigurationError, MetaError
 from datahub.configuration.connection_resolver import auto_connection_resolver
-from datahub.configuration.oauth import OAuthConfiguration, OAuthIdentityProvider
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
+from datahub.ingestion.api.closeable import Closeable
 from datahub.ingestion.source.snowflake.constants import (
     CLIENT_PREFETCH_THREADS,
     CLIENT_SESSION_KEEP_ALIVE,
 )
-from datahub.ingestion.source.sql.oauth_generator import OAuthTokenGenerator
+from datahub.ingestion.source.snowflake.oauth_config import (
+    OAuthConfiguration,
+    OAuthIdentityProvider,
+)
+from datahub.ingestion.source.snowflake.oauth_generator import OAuthTokenGenerator
 from datahub.ingestion.source.sql.sql_config import make_sqlalchemy_uri
 from datahub.utilities.config_clean import (
     remove_protocol,
@@ -39,6 +43,7 @@
     "EXTERNAL_BROWSER_AUTHENTICATOR": EXTERNAL_BROWSER_AUTHENTICATOR,
     "KEY_PAIR_AUTHENTICATOR": KEY_PAIR_AUTHENTICATOR,
     "OAUTH_AUTHENTICATOR": OAUTH_AUTHENTICATOR,
+    "OAUTH_AUTHENTICATOR_TOKEN": OAUTH_AUTHENTICATOR,
 }
 
 _SNOWFLAKE_HOST_SUFFIX = ".snowflakecomputing.com"
@@ -100,6 +105,10 @@ class SnowflakeConnectionConfig(ConfigModel):
         description="Connect args to pass to Snowflake SqlAlchemy driver",
         exclude=True,
     )
+    token: Optional[str] = pydantic.Field(
+        default=None,
+        description="OAuth token from external identity provider. Not recommended for most use cases because it will not be able to refresh once expired.",
+    )
 
     def get_account(self) -> str:
         assert self.account_id
@@ -144,6 +153,18 @@ def authenticator_type_is_valid(cls, v, values):
         logger.info(f"using authenticator type '{v}'")
         return v
 
+    @pydantic.validator("token", always=True)
+    def validate_token_oauth_config(cls, v, values):
+        auth_type = values.get("authentication_type")
+        if auth_type == "OAUTH_AUTHENTICATOR_TOKEN":
+            if not v:
+                raise ValueError("Token required for OAUTH_AUTHENTICATOR_TOKEN.")
+        elif v is not None:
+            raise ValueError(
+                "Token can only be provided when using OAUTH_AUTHENTICATOR_TOKEN"
+            )
+        return v
+
     @staticmethod
     def _check_oauth_config(oauth_config: Optional[OAuthConfiguration]) -> None:
         if oauth_config is None:
@@ -329,6 +350,17 @@ def get_native_connection(self) -> NativeSnowflakeConnection:
                 application=_APPLICATION_NAME,
                 **connect_args,
             )
+        elif self.authentication_type == "OAUTH_AUTHENTICATOR_TOKEN":
+            return snowflake.connector.connect(
+                user=self.username,
+                account=self.account_id,
+                authenticator="oauth",
+                token=self.token,  # Token generated externally and provided directly to the recipe
+                warehouse=self.warehouse,
+                role=self.role,
+                application=_APPLICATION_NAME,
+                **connect_args,
+            )
         elif self.authentication_type == "OAUTH_AUTHENTICATOR":
             return self.get_oauth_connection()
         elif self.authentication_type == "KEY_PAIR_AUTHENTICATOR":
@@ -364,7 +396,7 @@ def get_connection(self) -> "SnowflakeConnection":
             ) from e
 
 
-class SnowflakeConnection:
+class SnowflakeConnection(Closeable):
     _connection: NativeSnowflakeConnection
 
     def __init__(self, connection: NativeSnowflakeConnection):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
index 6f9c9259b2784..b815a6584379a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
@@ -4,7 +4,7 @@
 from datetime import datetime
 from typing import Any, Collection, Iterable, List, Optional, Set, Tuple, Type
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, Field, validator
 
 from datahub.configuration.datetimes import parse_absolute_time
 from datahub.ingestion.api.closeable import Closeable
@@ -31,14 +31,16 @@
 )
 from datahub.metadata.schema_classes import DatasetLineageTypeClass, UpstreamClass
 from datahub.sql_parsing.sql_parsing_aggregator import (
-    ColumnLineageInfo,
-    ColumnRef,
     KnownLineageMapping,
     KnownQueryLineageInfo,
     SqlParsingAggregator,
     UrnStr,
 )
-from datahub.sql_parsing.sqlglot_lineage import DownstreamColumnRef
+from datahub.sql_parsing.sqlglot_lineage import (
+    ColumnLineageInfo,
+    ColumnRef,
+    DownstreamColumnRef,
+)
 from datahub.utilities.perf_timer import PerfTimer
 from datahub.utilities.time import ts_millis_to_datetime
 
@@ -70,8 +72,8 @@ class ColumnUpstreamJob(BaseModel):
 
 
 class ColumnUpstreamLineage(BaseModel):
-    column_name: str
-    upstreams: List[ColumnUpstreamJob]
+    column_name: Optional[str]
+    upstreams: List[ColumnUpstreamJob] = Field(default_factory=list)
 
 
 class UpstreamTableNode(BaseModel):
@@ -234,7 +236,6 @@ def populate_known_query_lineage(
     def get_known_query_lineage(
         self, query: Query, dataset_name: str, db_row: UpstreamLineageEdge
     ) -> Optional[KnownQueryLineageInfo]:
-
         if not db_row.UPSTREAM_TABLES:
             return None
 
@@ -264,64 +265,17 @@ def _populate_external_upstreams(self, discovered_tables: List[str]) -> None:
         with PerfTimer() as timer:
             self.report.num_external_table_edges_scanned = 0
 
-            for (
-                known_lineage_mapping
-            ) in self._populate_external_lineage_from_copy_history(discovered_tables):
-                self.sql_aggregator.add(known_lineage_mapping)
-            logger.info(
-                "Done populating external lineage from copy history. "
-                f"Found {self.report.num_external_table_edges_scanned} external lineage edges so far."
-            )
-
-            for (
-                known_lineage_mapping
-            ) in self._populate_external_lineage_from_show_query(discovered_tables):
-                self.sql_aggregator.add(known_lineage_mapping)
-
-            logger.info(
-                "Done populating external lineage from show external tables. "
-                f"Found {self.report.num_external_table_edges_scanned} external lineage edges so far."
-            )
+            for entry in self._get_copy_history_lineage(discovered_tables):
+                self.sql_aggregator.add(entry)
+            logger.info("Done populating external lineage from copy history. ")
 
             self.report.external_lineage_queries_secs = timer.elapsed_seconds()
 
-    # Handles the case for explicitly created external tables.
-    # NOTE: Snowflake does not log this information to the access_history table.
-    def _populate_external_lineage_from_show_query(
-        self, discovered_tables: List[str]
-    ) -> Iterable[KnownLineageMapping]:
-        external_tables_query: str = SnowflakeQuery.show_external_tables()
-        try:
-            for db_row in self.connection.query(external_tables_query):
-                key = self.identifiers.get_dataset_identifier(
-                    db_row["name"], db_row["schema_name"], db_row["database_name"]
-                )
-
-                if key not in discovered_tables:
-                    continue
-                if db_row["location"].startswith("s3://"):
-                    yield KnownLineageMapping(
-                        upstream_urn=make_s3_urn_for_lineage(
-                            db_row["location"], self.config.env
-                        ),
-                        downstream_urn=self.identifiers.gen_dataset_urn(key),
-                    )
-                    self.report.num_external_table_edges_scanned += 1
-
-                self.report.num_external_table_edges_scanned += 1
-        except Exception as e:
-            logger.debug(e, exc_info=e)
-            self.structured_reporter.warning(
-                "Error populating external table lineage from Snowflake",
-                exc=e,
-            )
-            self.report_status(EXTERNAL_LINEAGE, False)
-
     # Handles the case where a table is populated from an external stage/s3 location via copy.
     # Eg: copy into category_english from @external_s3_stage;
     # Eg: copy into category_english from 's3://acryl-snow-demo-olist/olist_raw_data/category_english'credentials=(aws_key_id='...' aws_secret_key='...')  pattern='.*.csv';
     # NOTE: Snowflake does not log this information to the access_history table.
-    def _populate_external_lineage_from_copy_history(
+    def _get_copy_history_lineage(
         self, discovered_tables: List[str]
     ) -> Iterable[KnownLineageMapping]:
         query: str = SnowflakeQuery.copy_lineage_history(
@@ -412,9 +366,14 @@ def _process_upstream_lineage_row(
             return UpstreamLineageEdge.parse_obj(db_row)
         except Exception as e:
             self.report.num_upstream_lineage_edge_parsing_failed += 1
+            upstream_tables = db_row.get("UPSTREAM_TABLES")
+            downstream_table = db_row.get("DOWNSTREAM_TABLE_NAME")
             self.structured_reporter.warning(
                 "Failed to parse lineage edge",
-                context=db_row.get("DOWNSTREAM_TABLE_NAME") or None,
+                # Tricky: sometimes the full row data is too large, and so the context
+                # message gets truncated. By pulling out the upstreams and downstream
+                # list, we can at least get the important fields if truncation does occur.
+                context=f"Upstreams: {upstream_tables} Downstream: {downstream_table} Full row: {db_row}",
                 exc=e,
             )
             return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py
index 1445d02aa49db..174aad0bddd4a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py
@@ -1,3 +1,4 @@
+import contextlib
 import dataclasses
 import functools
 import json
@@ -17,6 +18,7 @@
     BaseTimeWindowConfig,
     BucketDuration,
 )
+from datahub.ingestion.api.closeable import Closeable
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.report import Report
 from datahub.ingestion.api.source import Source, SourceReport
@@ -50,6 +52,8 @@
     PreparsedQuery,
     SqlAggregatorReport,
     SqlParsingAggregator,
+    TableRename,
+    TableSwap,
 )
 from datahub.sql_parsing.sql_parsing_common import QueryType
 from datahub.sql_parsing.sqlglot_lineage import (
@@ -114,6 +118,8 @@ class SnowflakeQueriesExtractorReport(Report):
     audit_log_load_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer)
     sql_aggregator: Optional[SqlAggregatorReport] = None
 
+    num_ddl_queries_dropped: int = 0
+
 
 @dataclass
 class SnowflakeQueriesSourceReport(SourceReport):
@@ -121,7 +127,7 @@ class SnowflakeQueriesSourceReport(SourceReport):
     queries_extractor: Optional[SnowflakeQueriesExtractorReport] = None
 
 
-class SnowflakeQueriesExtractor(SnowflakeStructuredReportMixin):
+class SnowflakeQueriesExtractor(SnowflakeStructuredReportMixin, Closeable):
     def __init__(
         self,
         connection: SnowflakeConnection,
@@ -143,28 +149,33 @@ def __init__(
 
         self._structured_report = structured_report
 
-        self.aggregator = SqlParsingAggregator(
-            platform=self.identifiers.platform,
-            platform_instance=self.identifiers.identifier_config.platform_instance,
-            env=self.identifiers.identifier_config.env,
-            schema_resolver=schema_resolver,
-            graph=graph,
-            eager_graph_load=False,
-            generate_lineage=self.config.include_lineage,
-            generate_queries=self.config.include_queries,
-            generate_usage_statistics=self.config.include_usage_statistics,
-            generate_query_usage_statistics=self.config.include_query_usage_statistics,
-            usage_config=BaseUsageConfig(
-                bucket_duration=self.config.window.bucket_duration,
-                start_time=self.config.window.start_time,
-                end_time=self.config.window.end_time,
-                user_email_pattern=self.config.user_email_pattern,
-                # TODO make the rest of the fields configurable
-            ),
-            generate_operations=self.config.include_operations,
-            is_temp_table=self.is_temp_table,
-            is_allowed_table=self.is_allowed_table,
-            format_queries=False,
+        # The exit stack helps ensure that we close all the resources we open.
+        self._exit_stack = contextlib.ExitStack()
+
+        self.aggregator: SqlParsingAggregator = self._exit_stack.enter_context(
+            SqlParsingAggregator(
+                platform=self.identifiers.platform,
+                platform_instance=self.identifiers.identifier_config.platform_instance,
+                env=self.identifiers.identifier_config.env,
+                schema_resolver=schema_resolver,
+                graph=graph,
+                eager_graph_load=False,
+                generate_lineage=self.config.include_lineage,
+                generate_queries=self.config.include_queries,
+                generate_usage_statistics=self.config.include_usage_statistics,
+                generate_query_usage_statistics=self.config.include_query_usage_statistics,
+                usage_config=BaseUsageConfig(
+                    bucket_duration=self.config.window.bucket_duration,
+                    start_time=self.config.window.start_time,
+                    end_time=self.config.window.end_time,
+                    user_email_pattern=self.config.user_email_pattern,
+                    # TODO make the rest of the fields configurable
+                ),
+                generate_operations=self.config.include_operations,
+                is_temp_table=self.is_temp_table,
+                is_allowed_table=self.is_allowed_table,
+                format_queries=False,
+            )
         )
         self.report.sql_aggregator = self.aggregator.report
 
@@ -218,7 +229,9 @@ def get_workunits_internal(
         audit_log_file = self.local_temp_path / "audit_log.sqlite"
         use_cached_audit_log = audit_log_file.exists()
 
-        queries: FileBackedList[Union[KnownLineageMapping, PreparsedQuery]]
+        queries: FileBackedList[
+            Union[KnownLineageMapping, PreparsedQuery, TableRename, TableSwap]
+        ]
         if use_cached_audit_log:
             logger.info("Using cached audit log")
             shared_connection = ConnectionWrapper(audit_log_file)
@@ -228,15 +241,12 @@ def get_workunits_internal(
 
             shared_connection = ConnectionWrapper(audit_log_file)
             queries = FileBackedList(shared_connection)
-            entry: Union[KnownLineageMapping, PreparsedQuery]
+            entry: Union[KnownLineageMapping, PreparsedQuery, TableRename, TableSwap]
 
             with self.report.copy_history_fetch_timer:
                 for entry in self.fetch_copy_history():
                     queries.append(entry)
 
-            # TODO: Add "show external tables" lineage to the main schema extractor.
-            # Because it's not a time-based thing, it doesn't really make sense in the snowflake-queries extractor.
-
             with self.report.query_log_fetch_timer:
                 for entry in self.fetch_query_log():
                     queries.append(entry)
@@ -248,6 +258,10 @@ def get_workunits_internal(
                 self.aggregator.add(query)
 
         yield from auto_workunit(self.aggregator.gen_metadata())
+        if not use_cached_audit_log:
+            queries.close()
+            shared_connection.close()
+            audit_log_file.unlink(missing_ok=True)
 
     def fetch_copy_history(self) -> Iterable[KnownLineageMapping]:
         # Derived from _populate_external_lineage_from_copy_history.
@@ -285,7 +299,7 @@ def fetch_copy_history(self) -> Iterable[KnownLineageMapping]:
 
     def fetch_query_log(
         self,
-    ) -> Iterable[PreparsedQuery]:
+    ) -> Iterable[Union[PreparsedQuery, TableRename, TableSwap]]:
         query_log_query = _build_enriched_query_log_query(
             start_time=self.config.window.start_time,
             end_time=self.config.window.end_time,
@@ -313,12 +327,16 @@ def fetch_query_log(
                         exc=e,
                     )
                 else:
-                    yield entry
+                    if entry:
+                        yield entry
 
-    def _parse_audit_log_row(self, row: Dict[str, Any]) -> PreparsedQuery:
+    def _parse_audit_log_row(
+        self, row: Dict[str, Any]
+    ) -> Optional[Union[TableRename, TableSwap, PreparsedQuery]]:
         json_fields = {
             "DIRECT_OBJECTS_ACCESSED",
             "OBJECTS_MODIFIED",
+            "OBJECT_MODIFIED_BY_DDL",
         }
 
         res = {}
@@ -330,7 +348,26 @@ def _parse_audit_log_row(self, row: Dict[str, Any]) -> PreparsedQuery:
 
         direct_objects_accessed = res["direct_objects_accessed"]
         objects_modified = res["objects_modified"]
-
+        object_modified_by_ddl = res["object_modified_by_ddl"]
+
+        if object_modified_by_ddl and not objects_modified:
+            known_ddl_entry: Optional[Union[TableRename, TableSwap]] = None
+            with self.structured_reporter.report_exc(
+                "Error fetching ddl lineage from Snowflake"
+            ):
+                known_ddl_entry = self.parse_ddl_query(
+                    res["query_text"],
+                    res["session_id"],
+                    res["query_start_time"],
+                    object_modified_by_ddl,
+                )
+            if known_ddl_entry:
+                return known_ddl_entry
+            elif direct_objects_accessed:
+                # Unknown ddl relevant for usage. We want to continue execution here
+                pass
+            else:
+                return None
         upstreams = []
         column_usage = {}
 
@@ -426,6 +463,53 @@ def _parse_audit_log_row(self, row: Dict[str, Any]) -> PreparsedQuery:
         )
         return entry
 
+    def parse_ddl_query(
+        self,
+        query: str,
+        session_id: str,
+        timestamp: datetime,
+        object_modified_by_ddl: dict,
+    ) -> Optional[Union[TableRename, TableSwap]]:
+        timestamp = timestamp.astimezone(timezone.utc)
+        if object_modified_by_ddl[
+            "operationType"
+        ] == "ALTER" and object_modified_by_ddl["properties"].get("swapTargetName"):
+            urn1 = self.identifiers.gen_dataset_urn(
+                self.identifiers.get_dataset_identifier_from_qualified_name(
+                    object_modified_by_ddl["objectName"]
+                )
+            )
+
+            urn2 = self.identifiers.gen_dataset_urn(
+                self.identifiers.get_dataset_identifier_from_qualified_name(
+                    object_modified_by_ddl["properties"]["swapTargetName"]["value"]
+                )
+            )
+
+            return TableSwap(urn1, urn2, query, session_id, timestamp)
+        elif object_modified_by_ddl[
+            "operationType"
+        ] == "RENAME_TABLE" and object_modified_by_ddl["properties"].get("objectName"):
+            original_un = self.identifiers.gen_dataset_urn(
+                self.identifiers.get_dataset_identifier_from_qualified_name(
+                    object_modified_by_ddl["objectName"]
+                )
+            )
+
+            new_urn = self.identifiers.gen_dataset_urn(
+                self.identifiers.get_dataset_identifier_from_qualified_name(
+                    object_modified_by_ddl["properties"]["objectName"]["value"]
+                )
+            )
+
+            return TableRename(original_un, new_urn, query, session_id, timestamp)
+        else:
+            self.report.num_ddl_queries_dropped += 1
+            return None
+
+    def close(self) -> None:
+        self._exit_stack.close()
+
 
 class SnowflakeQueriesSource(Source):
     def __init__(self, ctx: PipelineContext, config: SnowflakeQueriesSourceConfig):
@@ -468,6 +552,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
     def get_report(self) -> SnowflakeQueriesSourceReport:
         return self.report
 
+    def close(self) -> None:
+        self.connection.close()
+        self.queries_extractor.close()
+
 
 # Make sure we don't try to generate too much info for a single query.
 _MAX_TABLES_PER_QUERY = 20
@@ -524,6 +612,7 @@ def _build_enriched_query_log_query(
         user_name,
         direct_objects_accessed,
         objects_modified,
+        object_modified_by_ddl
     FROM
         snowflake.account_usage.access_history
     WHERE
@@ -545,8 +634,9 @@ def _build_enriched_query_log_query(
         ) as direct_objects_accessed,
         -- TODO: Drop the columns.baseSources subfield.
         FILTER(objects_modified, o -> o:objectDomain IN {SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER}) as objects_modified,
+        case when object_modified_by_ddl:objectDomain IN {SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER} then object_modified_by_ddl else null end as object_modified_by_ddl
     FROM raw_access_history
-    WHERE ( array_size(direct_objects_accessed) > 0 or array_size(objects_modified) > 0 )
+    WHERE ( array_size(direct_objects_accessed) > 0 or array_size(objects_modified) > 0 or object_modified_by_ddl is not null )
 )
 , query_access_history AS (
     SELECT
@@ -568,6 +658,7 @@ def _build_enriched_query_log_query(
         q.role_name AS "ROLE_NAME",
         a.direct_objects_accessed,
         a.objects_modified,
+        a.object_modified_by_ddl
     FROM deduplicated_queries q
     JOIN filtered_access_history a USING (query_id)
 )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py
index 662e1cc2509ea..97c398c1962d6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py
@@ -129,10 +129,12 @@ def tables_for_database(db_name: Optional[str]) -> str:
         row_count AS "ROW_COUNT",
         bytes AS "BYTES",
         clustering_key AS "CLUSTERING_KEY",
-        auto_clustering_on AS "AUTO_CLUSTERING_ON"
+        auto_clustering_on AS "AUTO_CLUSTERING_ON",
+        is_dynamic AS "IS_DYNAMIC",
+        is_iceberg AS "IS_ICEBERG"
         FROM {db_clause}information_schema.tables t
         WHERE table_schema != 'INFORMATION_SCHEMA'
-        and table_type in ( 'BASE TABLE', 'EXTERNAL TABLE')
+        and table_type in ( 'BASE TABLE', 'EXTERNAL TABLE', 'HYBRID TABLE')
         order by table_schema, table_name"""
 
     @staticmethod
@@ -149,10 +151,12 @@ def tables_for_schema(schema_name: str, db_name: Optional[str]) -> str:
         row_count AS "ROW_COUNT",
         bytes AS "BYTES",
         clustering_key AS "CLUSTERING_KEY",
-        auto_clustering_on AS "AUTO_CLUSTERING_ON"
+        auto_clustering_on AS "AUTO_CLUSTERING_ON",
+        is_dynamic AS "IS_DYNAMIC",
+        is_iceberg AS "IS_ICEBERG"
         FROM {db_clause}information_schema.tables t
         where table_schema='{schema_name}'
-        and table_type in ('BASE TABLE', 'EXTERNAL TABLE')
+        and table_type in ('BASE TABLE', 'EXTERNAL TABLE', 'HYBRID TABLE')
         order by table_schema, table_name"""
 
     @staticmethod
@@ -233,6 +237,19 @@ def show_views_for_database(
 LIMIT {limit} {from_clause};
 """
 
+    @staticmethod
+    def get_secure_view_definitions() -> str:
+        # https://docs.snowflake.com/en/sql-reference/account-usage/views
+        return """
+            SELECT
+                TABLE_CATALOG as "TABLE_CATALOG",
+                TABLE_SCHEMA as "TABLE_SCHEMA",
+                TABLE_NAME as "TABLE_NAME",
+                VIEW_DEFINITION as "VIEW_DEFINITION"
+            FROM SNOWFLAKE.ACCOUNT_USAGE.VIEWS
+            WHERE IS_SECURE = 'YES' AND VIEW_DEFINITION !='' AND DELETED IS NULL
+        """
+
     @staticmethod
     def columns_for_schema(
         schema_name: str,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py
index 80b6be36e5ffa..030b2d43be81f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py
@@ -5,7 +5,7 @@
 from datahub.ingestion.api.report import Report
 from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin
 from datahub.ingestion.source.snowflake.constants import SnowflakeEdition
-from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionReport,
 )
@@ -59,7 +59,7 @@ class SnowflakeUsageReport:
 
 
 @dataclass
-class SnowflakeReport(ProfilingSqlReport, BaseTimeWindowReport):
+class SnowflakeReport(SQLSourceReport, BaseTimeWindowReport):
     num_table_to_table_edges_scanned: int = 0
     num_table_to_view_edges_scanned: int = 0
     num_view_to_table_edges_scanned: int = 0
@@ -113,6 +113,7 @@ class SnowflakeV2Report(
     external_lineage_queries_secs: float = -1
     num_tables_with_known_upstreams: int = 0
     num_upstream_lineage_edge_parsing_failed: int = 0
+    num_secure_views_missing_definition: int = 0
 
     data_dictionary_cache: Optional["SnowflakeDataDictionary"] = None
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py
index 600292c2c9942..780effc82b016 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py
@@ -90,6 +90,12 @@ class SnowflakeTable(BaseTable):
     foreign_keys: List[SnowflakeFK] = field(default_factory=list)
     tags: Optional[List[SnowflakeTag]] = None
     column_tags: Dict[str, List[SnowflakeTag]] = field(default_factory=dict)
+    is_dynamic: bool = False
+    is_iceberg: bool = False
+
+    @property
+    def is_hybrid(self) -> bool:
+        return self.type is not None and self.type == "HYBRID TABLE"
 
 
 @dataclass
@@ -98,6 +104,7 @@ class SnowflakeView(BaseView):
     columns: List[SnowflakeColumn] = field(default_factory=list)
     tags: Optional[List[SnowflakeTag]] = None
     column_tags: Dict[str, List[SnowflakeTag]] = field(default_factory=dict)
+    is_secure: bool = False
 
 
 @dataclass
@@ -259,6 +266,22 @@ def get_schemas_for_database(self, db_name: str) -> List[SnowflakeSchema]:
             snowflake_schemas.append(snowflake_schema)
         return snowflake_schemas
 
+    @serialized_lru_cache(maxsize=1)
+    def get_secure_view_definitions(self) -> Dict[str, Dict[str, Dict[str, str]]]:
+        secure_view_definitions: Dict[str, Dict[str, Dict[str, str]]] = defaultdict(
+            lambda: defaultdict(lambda: defaultdict())
+        )
+        cur = self.connection.query(SnowflakeQuery.get_secure_view_definitions())
+        for view in cur:
+            db_name = view["TABLE_CATALOG"]
+            schema_name = view["TABLE_SCHEMA"]
+            view_name = view["TABLE_NAME"]
+            secure_view_definitions[db_name][schema_name][view_name] = view[
+                "VIEW_DEFINITION"
+            ]
+
+        return secure_view_definitions
+
     @serialized_lru_cache(maxsize=1)
     def get_tables_for_database(
         self, db_name: str
@@ -289,6 +312,8 @@ def get_tables_for_database(
                     rows_count=table["ROW_COUNT"],
                     comment=table["COMMENT"],
                     clustering_key=table["CLUSTERING_KEY"],
+                    is_dynamic=table.get("IS_DYNAMIC", "NO").upper() == "YES",
+                    is_iceberg=table.get("IS_ICEBERG", "NO").upper() == "YES",
                 )
             )
         return tables
@@ -313,6 +338,8 @@ def get_tables_for_schema(
                     rows_count=table["ROW_COUNT"],
                     comment=table["COMMENT"],
                     clustering_key=table["CLUSTERING_KEY"],
+                    is_dynamic=table.get("IS_DYNAMIC", "NO").upper() == "YES",
+                    is_iceberg=table.get("IS_ICEBERG", "NO").upper() == "YES",
                 )
             )
         return tables
@@ -356,6 +383,7 @@ def get_views_for_database(self, db_name: str) -> Dict[str, List[SnowflakeView]]
                         materialized=(
                             view.get("is_materialized", "false").lower() == "true"
                         ),
+                        is_secure=(view.get("is_secure", "false").lower() == "true"),
                     )
                 )
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py
index d4442749a0622..4b72b09fafe2d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py
@@ -16,6 +16,7 @@
     ClassificationHandler,
     classification_workunit_processor,
 )
+from datahub.ingestion.source.aws.s3_util import make_s3_urn_for_lineage
 from datahub.ingestion.source.common.subtypes import (
     DatasetContainerSubTypes,
     DatasetSubTypes,
@@ -35,6 +36,7 @@
 )
 from datahub.ingestion.source.snowflake.snowflake_data_reader import SnowflakeDataReader
 from datahub.ingestion.source.snowflake.snowflake_profiler import SnowflakeProfiler
+from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery
 from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report
 from datahub.ingestion.source.snowflake.snowflake_schema import (
     SCHEMA_PARALLELISM,
@@ -65,6 +67,7 @@
     get_domain_wu,
 )
 from datahub.ingestion.source_report.ingestion_stage import (
+    EXTERNAL_TABLE_DDL_LINEAGE,
     METADATA_EXTRACTION,
     PROFILING,
 )
@@ -96,13 +99,17 @@
     TimeType,
 )
 from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties
-from datahub.sql_parsing.sql_parsing_aggregator import SqlParsingAggregator
+from datahub.sql_parsing.sql_parsing_aggregator import (
+    KnownLineageMapping,
+    SqlParsingAggregator,
+)
 from datahub.utilities.registries.domain_registry import DomainRegistry
 from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor
 
 logger = logging.getLogger(__name__)
 
 # https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html
+# TODO: Move to the standardized types in sql_types.py
 SNOWFLAKE_FIELD_TYPE_MAPPINGS = {
     "DATE": DateType,
     "BIGINT": NumberType,
@@ -179,7 +186,8 @@ def __init__(
 
         # These are populated as side-effects of get_workunits_internal.
         self.databases: List[SnowflakeDatabase] = []
-        self.aggregator: Optional[SqlParsingAggregator] = aggregator
+
+        self.aggregator = aggregator
 
     def get_connection(self) -> SnowflakeConnection:
         return self.connection
@@ -211,6 +219,19 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 self.report.set_ingestion_stage(snowflake_db.name, METADATA_EXTRACTION)
                 yield from self._process_database(snowflake_db)
 
+            self.report.set_ingestion_stage("*", EXTERNAL_TABLE_DDL_LINEAGE)
+            discovered_tables: List[str] = [
+                self.identifiers.get_dataset_identifier(
+                    table_name, schema.name, db.name
+                )
+                for db in self.databases
+                for schema in db.schemas
+                for table_name in schema.tables
+            ]
+            if self.aggregator:
+                for entry in self._external_tables_ddl_lineage(discovered_tables):
+                    self.aggregator.add(entry)
+
         except SnowflakePermissionError as e:
             self.structured_reporter.failure(
                 GENERIC_PERMISSION_ERROR_KEY,
@@ -423,6 +444,10 @@ def _process_schema(
                     view_identifier = self.identifiers.get_dataset_identifier(
                         view.name, schema_name, db_name
                     )
+                    if view.is_secure and not view.view_definition:
+                        view.view_definition = self.fetch_secure_view_definition(
+                            view.name, schema_name, db_name
+                        )
                     if view.view_definition:
                         self.aggregator.add_view_definition(
                             view_urn=self.identifiers.gen_dataset_urn(view_identifier),
@@ -430,6 +455,8 @@ def _process_schema(
                             default_db=db_name,
                             default_schema=schema_name,
                         )
+                    elif view.is_secure:
+                        self.report.num_secure_views_missing_definition += 1
 
             if self.config.include_technical_schema:
                 for view in views:
@@ -446,6 +473,25 @@ def _process_schema(
                 context=f"{db_name}.{schema_name}",
             )
 
+    def fetch_secure_view_definition(
+        self, table_name: str, schema_name: str, db_name: str
+    ) -> Optional[str]:
+        try:
+            view_definitions = self.data_dictionary.get_secure_view_definitions()
+            return view_definitions[db_name][schema_name][table_name]
+        except Exception as e:
+            if isinstance(e, SnowflakePermissionError):
+                error_msg = (
+                    "Failed to get secure views definitions. Please check permissions."
+                )
+            else:
+                error_msg = "Failed to get secure views definitions"
+            self.structured_reporter.warning(
+                error_msg,
+                exc=e,
+            )
+            return None
+
     def fetch_views_for_schema(
         self, snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str
     ) -> List[SnowflakeView]:
@@ -748,8 +794,21 @@ def get_dataset_properties(
     ) -> DatasetProperties:
         custom_properties = {}
 
-        if isinstance(table, SnowflakeTable) and table.clustering_key:
-            custom_properties["CLUSTERING_KEY"] = table.clustering_key
+        if isinstance(table, SnowflakeTable):
+            if table.clustering_key:
+                custom_properties["CLUSTERING_KEY"] = table.clustering_key
+
+            if table.is_hybrid:
+                custom_properties["IS_HYBRID"] = "true"
+
+            if table.is_dynamic:
+                custom_properties["IS_DYNAMIC"] = "true"
+
+            if table.is_iceberg:
+                custom_properties["IS_ICEBERG"] = "true"
+
+        if isinstance(table, SnowflakeView) and table.is_secure:
+            custom_properties["IS_SECURE"] = "true"
 
         return DatasetProperties(
             name=table.name,
@@ -1043,3 +1102,33 @@ def get_fk_constraints_for_table(
 
         # Access to table but none of its constraints - is this possible ?
         return constraints.get(table_name, [])
+
+    # Handles the case for explicitly created external tables.
+    # NOTE: Snowflake does not log this information to the access_history table.
+    def _external_tables_ddl_lineage(
+        self, discovered_tables: List[str]
+    ) -> Iterable[KnownLineageMapping]:
+        external_tables_query: str = SnowflakeQuery.show_external_tables()
+        try:
+            for db_row in self.connection.query(external_tables_query):
+                key = self.identifiers.get_dataset_identifier(
+                    db_row["name"], db_row["schema_name"], db_row["database_name"]
+                )
+
+                if key not in discovered_tables:
+                    continue
+                if db_row["location"].startswith("s3://"):
+                    yield KnownLineageMapping(
+                        upstream_urn=make_s3_urn_for_lineage(
+                            db_row["location"], self.config.env
+                        ),
+                        downstream_urn=self.identifiers.gen_dataset_urn(key),
+                    )
+                    self.report.num_external_table_edges_scanned += 1
+
+                self.report.num_external_table_edges_scanned += 1
+        except Exception as e:
+            self.structured_reporter.warning(
+                "External table ddl lineage extraction failed",
+                exc=e,
+            )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py
index 5e79530d2391b..8e0c97aa135e8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py
@@ -1,6 +1,6 @@
 import abc
 from functools import cached_property
-from typing import ClassVar, Literal, Optional, Tuple
+from typing import ClassVar, List, Literal, Optional, Tuple
 
 from datahub.configuration.pattern_utils import is_schema_allowed
 from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
@@ -119,7 +119,6 @@ def is_dataset_pattern_allowed(
     ) -> bool:
         if not dataset_type or not dataset_name:
             return True
-        dataset_params = dataset_name.split(".")
         if dataset_type.lower() not in (
             SnowflakeObjectDomain.TABLE,
             SnowflakeObjectDomain.EXTERNAL_TABLE,
@@ -131,6 +130,7 @@ def is_dataset_pattern_allowed(
         if _is_sys_table(dataset_name):
             return False
 
+        dataset_params = _split_qualified_name(dataset_name)
         if len(dataset_params) != 3:
             self.structured_reporter.info(
                 title="Unexpected dataset pattern",
@@ -184,6 +184,46 @@ def _is_sys_table(table_name: str) -> bool:
     return table_name.lower().startswith("sys$")
 
 
+def _split_qualified_name(qualified_name: str) -> List[str]:
+    """
+    Split a qualified name into its constituent parts.
+
+    >>> _split_qualified_name("db.my_schema.my_table")
+    ['db', 'my_schema', 'my_table']
+    >>> _split_qualified_name('"db"."my_schema"."my_table"')
+    ['db', 'my_schema', 'my_table']
+    >>> _split_qualified_name('TEST_DB.TEST_SCHEMA."TABLE.WITH.DOTS"')
+    ['TEST_DB', 'TEST_SCHEMA', 'TABLE.WITH.DOTS']
+    >>> _split_qualified_name('TEST_DB."SCHEMA.WITH.DOTS".MY_TABLE')
+    ['TEST_DB', 'SCHEMA.WITH.DOTS', 'MY_TABLE']
+    """
+
+    # Fast path - no quotes.
+    if '"' not in qualified_name:
+        return qualified_name.split(".")
+
+    # First pass - split on dots that are not inside quotes.
+    in_quote = False
+    parts: List[List[str]] = [[]]
+    for char in qualified_name:
+        if char == '"':
+            in_quote = not in_quote
+        elif char == "." and not in_quote:
+            parts.append([])
+        else:
+            parts[-1].append(char)
+
+    # Second pass - remove outer pairs of quotes.
+    result = []
+    for part in parts:
+        if len(part) > 2 and part[0] == '"' and part[-1] == '"':
+            part = part[1:-1]
+
+        result.append("".join(part))
+
+    return result
+
+
 # Qualified Object names from snowflake audit logs have quotes for for snowflake quoted identifiers,
 # For example "test-database"."test-schema".test_table
 # whereas we generate urns without quotes even for quoted identifiers for backward compatibility
@@ -192,7 +232,7 @@ def _is_sys_table(table_name: str) -> bool:
 def _cleanup_qualified_name(
     qualified_name: str, structured_reporter: SourceReport
 ) -> str:
-    name_parts = qualified_name.split(".")
+    name_parts = _split_qualified_name(qualified_name)
     if len(name_parts) != 3:
         if not _is_sys_table(qualified_name):
             structured_reporter.info(
@@ -203,9 +243,9 @@ def _cleanup_qualified_name(
             )
         return qualified_name.replace('"', "")
     return _combine_identifier_parts(
-        db_name=name_parts[0].strip('"'),
-        schema_name=name_parts[1].strip('"'),
-        table_name=name_parts[2].strip('"'),
+        db_name=name_parts[0],
+        schema_name=name_parts[1],
+        table_name=name_parts[2],
     )
 
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
index 0d7881f36554d..884e6c49f5b62 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
@@ -1,3 +1,4 @@
+import contextlib
 import functools
 import json
 import logging
@@ -16,6 +17,9 @@
     support_status,
 )
 from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
+from datahub.ingestion.api.incremental_properties_helper import (
+    auto_incremental_properties,
+)
 from datahub.ingestion.api.source import (
     CapabilityReport,
     MetadataWorkUnitProcessor,
@@ -47,11 +51,9 @@
     SnowflakeQueriesExtractor,
     SnowflakeQueriesExtractorConfig,
 )
+from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery
 from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report
-from datahub.ingestion.source.snowflake.snowflake_schema import (
-    SnowflakeDataDictionary,
-    SnowflakeQuery,
-)
+from datahub.ingestion.source.snowflake.snowflake_schema import SnowflakeDataDictionary
 from datahub.ingestion.source.snowflake.snowflake_schema_gen import (
     SnowflakeSchemaGenerator,
 )
@@ -149,15 +151,19 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config):
                 cached_domains=[k for k in self.config.domain], graph=self.ctx.graph
             )
 
-        self.connection = self.config.get_connection()
+        # The exit stack helps ensure that we close all the resources we open.
+        self._exit_stack = contextlib.ExitStack()
+
+        self.connection: SnowflakeConnection = self._exit_stack.enter_context(
+            self.config.get_connection()
+        )
 
         # For database, schema, tables, views, etc
         self.data_dictionary = SnowflakeDataDictionary(connection=self.connection)
         self.lineage_extractor: Optional[SnowflakeLineageExtractor] = None
-        self.aggregator: Optional[SqlParsingAggregator] = None
 
-        if self.config.use_queries_v2 or self.config.include_table_lineage:
-            self.aggregator = SqlParsingAggregator(
+        self.aggregator: SqlParsingAggregator = self._exit_stack.enter_context(
+            SqlParsingAggregator(
                 platform=self.identifiers.platform,
                 platform_instance=self.config.platform_instance,
                 env=self.config.env,
@@ -177,10 +183,10 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config):
                 generate_operations=False,
                 format_queries=self.config.format_sql_queries,
             )
-            self.report.sql_aggregator = self.aggregator.report
+        )
+        self.report.sql_aggregator = self.aggregator.report
 
         if self.config.include_table_lineage:
-            assert self.aggregator is not None
             redundant_lineage_run_skip_handler: Optional[
                 RedundantLineageRunSkipHandler
             ] = None
@@ -191,14 +197,16 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config):
                     pipeline_name=self.ctx.pipeline_name,
                     run_id=self.ctx.run_id,
                 )
-            self.lineage_extractor = SnowflakeLineageExtractor(
-                config,
-                self.report,
-                connection=self.connection,
-                filters=self.filters,
-                identifiers=self.identifiers,
-                redundant_run_skip_handler=redundant_lineage_run_skip_handler,
-                sql_aggregator=self.aggregator,
+            self.lineage_extractor = self._exit_stack.enter_context(
+                SnowflakeLineageExtractor(
+                    config,
+                    self.report,
+                    connection=self.connection,
+                    filters=self.filters,
+                    identifiers=self.identifiers,
+                    redundant_run_skip_handler=redundant_lineage_run_skip_handler,
+                    sql_aggregator=self.aggregator,
+                )
             )
 
         self.usage_extractor: Optional[SnowflakeUsageExtractor] = None
@@ -213,13 +221,15 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config):
                     pipeline_name=self.ctx.pipeline_name,
                     run_id=self.ctx.run_id,
                 )
-            self.usage_extractor = SnowflakeUsageExtractor(
-                config,
-                self.report,
-                connection=self.connection,
-                filter=self.filters,
-                identifiers=self.identifiers,
-                redundant_run_skip_handler=redundant_usage_run_skip_handler,
+            self.usage_extractor = self._exit_stack.enter_context(
+                SnowflakeUsageExtractor(
+                    config,
+                    self.report,
+                    connection=self.connection,
+                    filter=self.filters,
+                    identifiers=self.identifiers,
+                    redundant_run_skip_handler=redundant_usage_run_skip_handler,
+                )
             )
 
         self.profiling_state_handler: Optional[ProfilingHandler] = None
@@ -436,6 +446,9 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
             functools.partial(
                 auto_incremental_lineage, self.config.incremental_lineage
             ),
+            functools.partial(
+                auto_incremental_properties, self.config.incremental_properties
+            ),
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
             ).workunit_processor,
@@ -444,10 +457,6 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         self._snowflake_clear_ocsp_cache()
 
-        self.connection = self.config.get_connection()
-        if self.connection is None:
-            return
-
         self.inspect_session_metadata(self.connection)
 
         snowsight_url_builder = None
@@ -475,8 +484,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
         databases = schema_extractor.databases
 
-        # TODO: The checkpoint state for stale entity detection can be committed here.
-
         if self.config.shares:
             yield from SnowflakeSharesHandler(
                 self.config, self.report
@@ -513,7 +520,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
             schema_resolver = self.aggregator._schema_resolver
 
-            queries_extractor = SnowflakeQueriesExtractor(
+            queries_extractor: SnowflakeQueriesExtractor = SnowflakeQueriesExtractor(
                 connection=self.connection,
                 config=SnowflakeQueriesExtractorConfig(
                     window=self.config,
@@ -528,6 +535,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 identifiers=self.identifiers,
                 schema_resolver=schema_resolver,
                 discovered_tables=discovered_datasets,
+                graph=self.ctx.graph,
             )
 
             # TODO: This is slightly suboptimal because we create two SqlParsingAggregator instances with different configs
@@ -535,6 +543,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             # it should be pretty straightforward to refactor this and only initialize the aggregator once.
             self.report.queries_extractor = queries_extractor.report
             yield from queries_extractor.get_workunits_internal()
+            queries_extractor.close()
 
         else:
             if self.config.include_table_lineage and self.lineage_extractor:
@@ -723,7 +732,4 @@ def _snowflake_clear_ocsp_cache(self) -> None:
     def close(self) -> None:
         super().close()
         StatefulIngestionSourceBase.close(self)
-        if self.lineage_extractor:
-            self.lineage_extractor.close()
-        if self.usage_extractor:
-            self.usage_extractor.close()
+        self._exit_stack.close()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
index 71cfd0268ee6b..6f7decc79b1df 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
@@ -26,6 +26,7 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.source import StructuredLogLevel
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.aws.s3_util import make_s3_urn
 from datahub.ingestion.source.common.subtypes import DatasetContainerSubTypes
@@ -35,6 +36,7 @@
     register_custom_type,
 )
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig, make_sqlalchemy_uri
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source.sql.sql_utils import (
     add_table_to_schema_container,
     gen_database_container,
@@ -48,6 +50,15 @@
     get_schema_fields_for_sqlalchemy_column,
 )
 
+try:
+    from typing_extensions import override
+except ImportError:
+    _F = typing.TypeVar("_F", bound=typing.Callable[..., typing.Any])
+
+    def override(f: _F, /) -> _F:  # noqa: F811
+        return f
+
+
 logger = logging.getLogger(__name__)
 
 assert STRUCT, "required type modules are not available"
@@ -322,12 +333,15 @@ class AthenaSource(SQLAlchemySource):
     - Profiling when enabled.
     """
 
-    table_partition_cache: Dict[str, Dict[str, Partitionitem]] = {}
+    config: AthenaConfig
+    report: SQLSourceReport
 
     def __init__(self, config, ctx):
         super().__init__(config, ctx, "athena")
         self.cursor: Optional[BaseCursor] = None
 
+        self.table_partition_cache: Dict[str, Dict[str, Partitionitem]] = {}
+
     @classmethod
     def create(cls, config_dict, ctx):
         config = AthenaConfig.parse_obj(config_dict)
@@ -452,6 +466,7 @@ def add_table_to_schema_container(
         )
 
     # It seems like database/schema filter in the connection string does not work and this to work around that
+    @override
     def get_schema_names(self, inspector: Inspector) -> List[str]:
         athena_config = typing.cast(AthenaConfig, self.config)
         schemas = inspector.get_schema_names()
@@ -459,34 +474,42 @@ def get_schema_names(self, inspector: Inspector) -> List[str]:
             return [schema for schema in schemas if schema == athena_config.database]
         return schemas
 
-    # Overwrite to get partitions
+    @classmethod
+    def _casted_partition_key(cls, key: str) -> str:
+        # We need to cast the partition keys to a VARCHAR, since otherwise
+        # Athena may throw an error during concatenation / comparison.
+        return f"CAST({key} as VARCHAR)"
+
+    @override
     def get_partitions(
         self, inspector: Inspector, schema: str, table: str
-    ) -> List[str]:
-        partitions = []
-
-        athena_config = typing.cast(AthenaConfig, self.config)
-
-        if not athena_config.extract_partitions:
-            return []
+    ) -> Optional[List[str]]:
+        if not self.config.extract_partitions:
+            return None
 
         if not self.cursor:
-            return []
+            return None
 
         metadata: AthenaTableMetadata = self.cursor.get_table_metadata(
             table_name=table, schema_name=schema
         )
 
-        if metadata.partition_keys:
-            for key in metadata.partition_keys:
-                if key.name:
-                    partitions.append(key.name)
-
-            if not partitions:
-                return []
+        partitions = []
+        for key in metadata.partition_keys:
+            if key.name:
+                partitions.append(key.name)
+        if not partitions:
+            return []
 
-            # We create an artiificaial concatenated partition key to be able to query max partition easier
-            part_concat = "|| '-' ||".join(partitions)
+        with self.report.report_exc(
+            message="Failed to extract partition details",
+            context=f"{schema}.{table}",
+            level=StructuredLogLevel.WARN,
+        ):
+            # We create an artifical concatenated partition key to be able to query max partition easier
+            part_concat = " || '-' || ".join(
+                self._casted_partition_key(key) for key in partitions
+            )
             max_partition_query = f'select {",".join(partitions)} from "{schema}"."{table}$partitions" where {part_concat} = (select max({part_concat}) from "{schema}"."{table}$partitions")'
             ret = self.cursor.execute(max_partition_query)
             max_partition: Dict[str, str] = {}
@@ -500,9 +523,8 @@ def get_partitions(
                 partitions=partitions,
                 max_partition=max_partition,
             )
-            return partitions
 
-        return []
+        return partitions
 
     # Overwrite to modify the creation of schema fields
     def get_schema_fields_for_column(
@@ -551,7 +573,9 @@ def generate_partition_profiler_query(
         if partition and partition.max_partition:
             max_partition_filters = []
             for key, value in partition.max_partition.items():
-                max_partition_filters.append(f"CAST({key} as VARCHAR) = '{value}'")
+                max_partition_filters.append(
+                    f"{self._casted_partition_key(key)} = '{value}'"
+                )
             max_partition = str(partition.max_partition)
             return (
                 max_partition,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py b/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py
index 5356cee7f6ea3..76b72d8e37f74 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py
@@ -28,7 +28,6 @@ class CockroachDBConfig(PostgresConfig):
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
 @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
 class CockroachDBSource(PostgresSource):
-
     config: CockroachDBConfig
 
     def __init__(self, config: CockroachDBConfig, ctx: PipelineContext):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
index 59f301baf4016..fad54fda45378 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
@@ -1,7 +1,10 @@
 import json
 import logging
 import re
-from typing import Any, Dict, Iterable, List, Optional, Union
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
+from urllib.parse import urlparse
 
 from pydantic.class_validators import validator
 from pydantic.fields import Field
@@ -11,7 +14,12 @@
 from pyhive.sqlalchemy_hive import HiveDate, HiveDecimal, HiveDialect, HiveTimestamp
 from sqlalchemy.engine.reflection import Inspector
 
-from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
+from datahub.emitter.mce_builder import (
+    make_data_platform_urn,
+    make_dataplatform_instance_urn,
+    make_dataset_urn_with_platform_instance,
+    make_schema_field_urn,
+)
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.decorators import (
     SourceCapability,
@@ -29,14 +37,24 @@
     TwoTierSQLAlchemyConfig,
     TwoTierSQLAlchemySource,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+from datahub.metadata.schema_classes import (
+    DataPlatformInstanceClass,
+    DatasetLineageTypeClass,
+    DatasetPropertiesClass,
     DateTypeClass,
+    FineGrainedLineageClass,
+    FineGrainedLineageDownstreamTypeClass,
+    FineGrainedLineageUpstreamTypeClass,
     NullTypeClass,
     NumberTypeClass,
-    SchemaField,
+    OtherSchemaClass,
+    SchemaFieldClass,
+    SchemaMetadataClass,
     TimeTypeClass,
+    UpstreamClass,
+    UpstreamLineageClass,
+    ViewPropertiesClass,
 )
-from datahub.metadata.schema_classes import ViewPropertiesClass
 from datahub.utilities import config_clean
 from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column
 
@@ -46,6 +64,511 @@
 register_custom_type(HiveTimestamp, TimeTypeClass)
 register_custom_type(HiveDecimal, NumberTypeClass)
 
+
+class StoragePlatform(Enum):
+    """Enumeration of storage platforms supported for lineage"""
+
+    S3 = "s3"
+    AZURE = "abs"
+    GCS = "gcs"
+    DBFS = "dbfs"
+    LOCAL = "file"
+    HDFS = "hdfs"
+
+
+# Mapping of URL schemes to storage platforms
+STORAGE_SCHEME_MAPPING = {
+    # S3 and derivatives
+    "s3": StoragePlatform.S3,
+    "s3a": StoragePlatform.S3,
+    "s3n": StoragePlatform.S3,
+    # Azure and derivatives
+    "abfs": StoragePlatform.AZURE,
+    "abfss": StoragePlatform.AZURE,
+    "adl": StoragePlatform.AZURE,
+    "adls": StoragePlatform.AZURE,
+    "wasb": StoragePlatform.AZURE,
+    "wasbs": StoragePlatform.AZURE,
+    # GCS and derivatives
+    "gs": StoragePlatform.GCS,
+    "gcs": StoragePlatform.GCS,
+    # DBFS
+    "dbfs": StoragePlatform.DBFS,
+    # Local filesystem
+    "file": StoragePlatform.LOCAL,
+    # HDFS
+    "hdfs": StoragePlatform.HDFS,
+}
+
+
+class StoragePathParser:
+    """Parser for storage paths with platform-specific logic"""
+
+    @staticmethod
+    def parse_storage_location(location: str) -> Optional[Tuple[StoragePlatform, str]]:
+        """
+        Parse a storage location into platform and normalized path.
+
+        Args:
+            location: Storage location URI (e.g., s3://bucket/path, abfss://container@account.dfs.core.windows.net/path)
+
+        Returns:
+            Tuple of (StoragePlatform, normalized_path) if valid, None if invalid
+        """
+
+        try:
+            # Handle special case for local files with no scheme
+            if location.startswith("/"):
+                return StoragePlatform.LOCAL, location
+
+            # Parse the URI
+            parsed = urlparse(location)
+            scheme = parsed.scheme.lower()
+
+            if not scheme:
+                return None
+
+            # Look up the platform
+            platform = STORAGE_SCHEME_MAPPING.get(scheme)
+            if not platform:
+                return None
+
+            # Get normalized path based on platform
+            if platform == StoragePlatform.S3:
+                # For S3, combine bucket and path
+                path = f"{parsed.netloc}/{parsed.path.lstrip('/')}"
+
+            elif platform == StoragePlatform.AZURE:
+                if scheme in ("abfs", "abfss"):
+                    # Format: abfss://container@account.dfs.core.windows.net/path
+                    container = parsed.netloc.split("@")[0]
+                    path = f"{container}/{parsed.path.lstrip('/')}"
+                else:
+                    # Handle other Azure schemes
+                    path = f"{parsed.netloc}/{parsed.path.lstrip('/')}"
+
+            elif platform == StoragePlatform.GCS:
+                # For GCS, combine bucket and path
+                path = f"{parsed.netloc}/{parsed.path.lstrip('/')}"
+
+            elif platform == StoragePlatform.DBFS:
+                # For DBFS, use path as-is
+                path = parsed.path.lstrip("/")
+
+            elif platform == StoragePlatform.LOCAL:
+                # For local files, use full path
+                path = f"{parsed.netloc}/{parsed.path.lstrip('/')}"
+
+            elif platform == StoragePlatform.HDFS:
+                # For HDFS, use full path
+                path = f"{parsed.netloc}/{parsed.path.lstrip('/')}"
+
+            else:
+                return None
+
+            # Clean up the path
+            path = path.rstrip("/")  # Remove trailing slashes
+            path = re.sub(r"/+", "/", path)  # Normalize multiple slashes
+            path = f"/{path}"
+
+            return platform, path
+
+        except Exception as exp:
+            logger.warning(f"Failed to parse storage location {location}: {exp}")
+            return None
+
+    @staticmethod
+    def get_platform_name(platform: StoragePlatform) -> str:
+        """Get the platform name to use in URNs"""
+
+        platform_names = {
+            StoragePlatform.S3: "s3",
+            StoragePlatform.AZURE: "adls",
+            StoragePlatform.GCS: "gcs",
+            StoragePlatform.DBFS: "dbfs",
+            StoragePlatform.LOCAL: "file",
+            StoragePlatform.HDFS: "hdfs",
+        }
+        return platform_names[platform]
+
+
+class HiveStorageLineageConfig:
+    """Configuration for Hive storage lineage."""
+
+    def __init__(
+        self,
+        emit_storage_lineage: bool,
+        hive_storage_lineage_direction: str,
+        include_column_lineage: bool,
+        storage_platform_instance: Optional[str],
+    ):
+        if hive_storage_lineage_direction.lower() not in ["upstream", "downstream"]:
+            raise ValueError(
+                "hive_storage_lineage_direction must be either upstream or downstream"
+            )
+
+        self.emit_storage_lineage = emit_storage_lineage
+        self.hive_storage_lineage_direction = hive_storage_lineage_direction.lower()
+        self.include_column_lineage = include_column_lineage
+        self.storage_platform_instance = storage_platform_instance
+
+
+@dataclass
+class HiveStorageSourceReport:
+    """Report for tracking storage lineage statistics"""
+
+    storage_locations_scanned: int = 0
+    filtered_locations: List[str] = Field(default_factory=list)
+    failed_locations: List[str] = Field(default_factory=list)
+
+    def report_location_scanned(self) -> None:
+        self.storage_locations_scanned += 1
+
+    def report_location_filtered(self, location: str) -> None:
+        self.filtered_locations.append(location)
+
+    def report_location_failed(self, location: str) -> None:
+        self.failed_locations.append(location)
+
+
+class HiveStorageLineage:
+    """Handles storage lineage for Hive tables"""
+
+    def __init__(
+        self,
+        config: HiveStorageLineageConfig,
+        env: str,
+        convert_urns_to_lowercase: bool = False,
+    ):
+        self.config = config
+        self.env = env
+        self.convert_urns_to_lowercase = convert_urns_to_lowercase
+        self.report = HiveStorageSourceReport()
+
+    def _make_dataset_platform_instance(
+        self,
+        platform: str,
+        instance: Optional[str],
+    ) -> DataPlatformInstanceClass:
+        """Create DataPlatformInstance aspect"""
+
+        return DataPlatformInstanceClass(
+            platform=make_data_platform_urn(platform),
+            instance=make_dataplatform_instance_urn(platform, instance)
+            if instance
+            else None,
+        )
+
+    def _make_storage_dataset_urn(
+        self,
+        storage_location: str,
+    ) -> Optional[Tuple[str, str]]:
+        """
+        Create storage dataset URN from location.
+        Returns tuple of (urn, platform) if successful, None otherwise.
+        """
+
+        platform_instance = None
+        storage_info = StoragePathParser.parse_storage_location(storage_location)
+        if not storage_info:
+            logger.debug(f"Could not parse storage location: {storage_location}")
+            return None
+
+        platform, path = storage_info
+        platform_name = StoragePathParser.get_platform_name(platform)
+
+        if self.convert_urns_to_lowercase:
+            platform_name = platform_name.lower()
+            path = path.lower()
+            if self.config.storage_platform_instance:
+                platform_instance = self.config.storage_platform_instance.lower()
+
+        try:
+            storage_urn = make_dataset_urn_with_platform_instance(
+                platform=platform_name,
+                name=path,
+                env=self.env,
+                platform_instance=platform_instance,
+            )
+            return storage_urn, platform_name
+        except Exception as exp:
+            logger.error(f"Failed to create URN for {platform_name}:{path}: {exp}")
+            return None
+
+    def _get_fine_grained_lineages(
+        self,
+        dataset_urn: str,
+        storage_urn: str,
+        dataset_schema: SchemaMetadataClass,
+        storage_schema: SchemaMetadataClass,
+    ) -> Iterable[FineGrainedLineageClass]:
+        """Generate column-level lineage between dataset and storage"""
+
+        if not self.config.include_column_lineage:
+            return
+
+        for dataset_field in dataset_schema.fields:
+            dataset_path = dataset_field.fieldPath
+
+            # Find matching field in storage schema
+            matching_field = next(
+                (f for f in storage_schema.fields if f.fieldPath == dataset_path),
+                None,
+            )
+
+            if matching_field:
+                if self.config.hive_storage_lineage_direction == "upstream":
+                    yield FineGrainedLineageClass(
+                        upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+                        upstreams=[
+                            make_schema_field_urn(
+                                parent_urn=storage_urn,
+                                field_path=matching_field.fieldPath,
+                            )
+                        ],
+                        downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD,
+                        downstreams=[
+                            make_schema_field_urn(
+                                parent_urn=dataset_urn,
+                                field_path=dataset_path,
+                            )
+                        ],
+                    )
+                else:
+                    yield FineGrainedLineageClass(
+                        upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+                        upstreams=[
+                            make_schema_field_urn(
+                                parent_urn=dataset_urn,
+                                field_path=dataset_path,
+                            )
+                        ],
+                        downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD,
+                        downstreams=[
+                            make_schema_field_urn(
+                                parent_urn=storage_urn,
+                                field_path=matching_field.fieldPath,
+                            )
+                        ],
+                    )
+
+    def _create_lineage_mcp(
+        self,
+        source_urn: str,
+        target_urn: str,
+        fine_grained_lineages: Optional[Iterable[FineGrainedLineageClass]] = None,
+    ) -> Iterable[MetadataWorkUnit]:
+        """Create lineage MCP between source and target datasets"""
+
+        lineages_list = (
+            list(fine_grained_lineages) if fine_grained_lineages is not None else None
+        )
+
+        upstream_lineage = UpstreamLineageClass(
+            upstreams=[
+                UpstreamClass(dataset=source_urn, type=DatasetLineageTypeClass.COPY)
+            ],
+            fineGrainedLineages=lineages_list,
+        )
+
+        yield MetadataWorkUnit(
+            id=f"{source_urn}-{target_urn}-lineage",
+            mcp=MetadataChangeProposalWrapper(
+                entityUrn=target_urn, aspect=upstream_lineage
+            ),
+        )
+
+    def get_storage_dataset_mcp(
+        self,
+        storage_location: str,
+        platform_instance: Optional[str] = None,
+        schema_metadata: Optional[SchemaMetadataClass] = None,
+    ) -> Iterable[MetadataWorkUnit]:
+        """
+        Generate MCPs for storage dataset if needed.
+        This creates the storage dataset entity in DataHub.
+        """
+
+        storage_info = StoragePathParser.parse_storage_location(
+            storage_location,
+        )
+        if not storage_info:
+            return
+
+        platform, path = storage_info
+        platform_name = StoragePathParser.get_platform_name(platform)
+
+        if self.convert_urns_to_lowercase:
+            platform_name = platform_name.lower()
+            path = path.lower()
+            if self.config.storage_platform_instance:
+                platform_instance = self.config.storage_platform_instance.lower()
+
+        try:
+            storage_urn = make_dataset_urn_with_platform_instance(
+                platform=platform_name,
+                name=path,
+                env=self.env,
+                platform_instance=platform_instance,
+            )
+
+            # Dataset properties
+            props = DatasetPropertiesClass(name=path)
+            yield MetadataWorkUnit(
+                id=f"storage-{storage_urn}-props",
+                mcp=MetadataChangeProposalWrapper(
+                    entityUrn=storage_urn,
+                    aspect=props,
+                ),
+            )
+
+            # Platform instance
+            platform_instance_aspect = self._make_dataset_platform_instance(
+                platform=platform_name,
+                instance=platform_instance,
+            )
+            yield MetadataWorkUnit(
+                id=f"storage-{storage_urn}-platform",
+                mcp=MetadataChangeProposalWrapper(
+                    entityUrn=storage_urn, aspect=platform_instance_aspect
+                ),
+            )
+
+            # Schema if available
+            if schema_metadata:
+                storage_schema = SchemaMetadataClass(
+                    schemaName=f"{platform.value}_schema",
+                    platform=f"urn:li:dataPlatform:{platform.value}",
+                    version=0,
+                    fields=schema_metadata.fields,
+                    hash="",
+                    platformSchema=OtherSchemaClass(rawSchema=""),
+                )
+                yield MetadataWorkUnit(
+                    id=f"storage-{storage_urn}-schema",
+                    mcp=MetadataChangeProposalWrapper(
+                        entityUrn=storage_urn, aspect=storage_schema
+                    ),
+                )
+
+        except Exception as e:
+            logger.error(
+                f"Failed to create storage dataset MCPs for {storage_location}: {e}"
+            )
+            return
+
+    def get_lineage_mcp(
+        self,
+        dataset_urn: str,
+        table: Dict[str, Any],
+        dataset_schema: Optional[SchemaMetadataClass] = None,
+    ) -> Iterable[MetadataWorkUnit]:
+        """
+        Generate lineage MCP for a Hive table to its storage location.
+
+        Args:
+            dataset_urn: URN of the Hive dataset
+            table: Hive table dictionary containing metadata
+            dataset_schema: Optional schema metadata for the Hive dataset
+
+        Returns:
+            MetadataWorkUnit containing the lineage MCP if successful
+        """
+
+        platform_instance = None
+
+        if not self.config.emit_storage_lineage:
+            return
+
+        # Get storage location from table
+        storage_location = table.get("StorageDescriptor", {}).get("Location")
+        if not storage_location:
+            return
+
+        # Create storage dataset URN
+        storage_info = self._make_storage_dataset_urn(storage_location)
+        if not storage_info:
+            self.report.report_location_failed(storage_location)
+            return
+
+        storage_urn, storage_platform = storage_info
+        self.report.report_location_scanned()
+
+        if self.config.storage_platform_instance:
+            platform_instance = self.config.storage_platform_instance.lower()
+
+        # Create storage dataset entity
+        yield from self.get_storage_dataset_mcp(
+            storage_location=storage_location,
+            platform_instance=platform_instance,
+            schema_metadata=dataset_schema,
+        )
+
+        # Get storage schema if available (implement based on storage system)
+        storage_schema = (
+            self._get_storage_schema(storage_location, dataset_schema)
+            if dataset_schema
+            else None
+        )
+
+        # Generate fine-grained lineage if schemas available
+        fine_grained_lineages = (
+            None
+            if not (dataset_schema and storage_schema)
+            else self._get_fine_grained_lineages(
+                dataset_urn, storage_urn, dataset_schema, storage_schema
+            )
+        )
+
+        # Create lineage MCP
+        if self.config.hive_storage_lineage_direction == "upstream":
+            yield from self._create_lineage_mcp(
+                source_urn=storage_urn,
+                target_urn=dataset_urn,
+                fine_grained_lineages=fine_grained_lineages,
+            )
+        else:
+            yield from self._create_lineage_mcp(
+                source_urn=dataset_urn,
+                target_urn=storage_urn,
+                fine_grained_lineages=fine_grained_lineages,
+            )
+
+    def _get_storage_schema(
+        self,
+        storage_location: str,
+        table_schema: Optional[SchemaMetadataClass] = None,
+    ) -> Optional[SchemaMetadataClass]:
+        """
+        Get schema metadata for storage location.
+        Currently supports:
+        - Delta tables
+        - Parquet files
+        - Spark tables
+
+        Returns:
+            SchemaMetadataClass if schema can be inferred, None otherwise
+        """
+
+        if not table_schema:
+            return None
+
+        storage_info = StoragePathParser.parse_storage_location(storage_location)
+        if not storage_info:
+            return None
+
+        platform, _ = storage_info
+
+        return SchemaMetadataClass(
+            schemaName=f"{platform.value}_schema",
+            platform=f"urn:li:dataPlatform:{platform.value}",
+            version=0,
+            fields=table_schema.fields,
+            hash="",
+            platformSchema=OtherSchemaClass(rawSchema=""),
+        )
+
+
 try:
     from databricks_dbapi.sqlalchemy_dialects.hive import DatabricksPyhiveDialect
     from pyhive.sqlalchemy_hive import _type_map
@@ -94,8 +617,8 @@ def dbapi_get_columns_patched(self, connection, table_name, schema=None, **kw):
     DatabricksPyhiveDialect.get_columns = dbapi_get_columns_patched
 except ModuleNotFoundError:
     pass
-except Exception as e:
-    logger.warning(f"Failed to patch method due to {e}")
+except Exception as exp:
+    logger.warning(f"Failed to patch method due to {exp}")
 
 
 @reflection.cache  # type: ignore
@@ -126,10 +649,48 @@ class HiveConfig(TwoTierSQLAlchemyConfig):
     # defaults
     scheme: str = Field(default="hive", hidden_from_docs=True)
 
+    # Overriding as table location lineage is richer implementation here than with include_table_location_lineage
+    include_table_location_lineage: bool = Field(default=False, hidden_from_docs=True)
+
+    emit_storage_lineage: bool = Field(
+        default=False,
+        description="Whether to emit storage-to-Hive lineage",
+    )
+    hive_storage_lineage_direction: str = Field(
+        default="upstream",
+        description="If 'upstream', storage is upstream to Hive. If 'downstream' storage is downstream to Hive",
+    )
+    include_column_lineage: bool = Field(
+        default=True,
+        description="When enabled, column-level lineage will be extracted from storage",
+    )
+    storage_platform_instance: Optional[str] = Field(
+        default=None,
+        description="Platform instance for the storage system",
+    )
+
     @validator("host_port")
     def clean_host_port(cls, v):
         return config_clean.remove_protocol(v)
 
+    @validator("hive_storage_lineage_direction")
+    def _validate_direction(cls, v: str) -> str:
+        """Validate the lineage direction."""
+        if v.lower() not in ["upstream", "downstream"]:
+            raise ValueError(
+                "storage_lineage_direction must be either upstream or downstream"
+            )
+        return v.lower()
+
+    def get_storage_lineage_config(self) -> HiveStorageLineageConfig:
+        """Convert base config parameters to HiveStorageLineageConfig"""
+        return HiveStorageLineageConfig(
+            emit_storage_lineage=self.emit_storage_lineage,
+            hive_storage_lineage_direction=self.hive_storage_lineage_direction,
+            include_column_lineage=self.include_column_lineage,
+            storage_platform_instance=self.storage_platform_instance,
+        )
+
 
 @platform_name("Hive")
 @config_class(HiveConfig)
@@ -151,12 +712,49 @@ class HiveSource(TwoTierSQLAlchemySource):
 
     def __init__(self, config, ctx):
         super().__init__(config, ctx, "hive")
+        self.storage_lineage = HiveStorageLineage(
+            config=config.get_storage_lineage_config(),
+            env=config.env,
+            convert_urns_to_lowercase=config.convert_urns_to_lowercase,
+        )
 
     @classmethod
     def create(cls, config_dict, ctx):
         config = HiveConfig.parse_obj(config_dict)
         return cls(config, ctx)
 
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        """Generate workunits for tables and their storage lineage."""
+        for wu in super().get_workunits_internal():
+            yield wu
+
+            if not isinstance(wu, MetadataWorkUnit):
+                continue
+
+            # Get dataset URN and required aspects using workunit methods
+            try:
+                dataset_urn = wu.get_urn()
+                dataset_props = wu.get_aspect_of_type(DatasetPropertiesClass)
+                schema_metadata = wu.get_aspect_of_type(SchemaMetadataClass)
+            except Exception as exp:
+                logger.warning(f"Failed to process workunit {wu.id}: {exp}")
+                continue
+
+            # Only proceed if we have the necessary properties
+            if dataset_props and dataset_props.customProperties:
+                table = {
+                    "StorageDescriptor": {
+                        "Location": dataset_props.customProperties.get("Location")
+                    }
+                }
+
+                if table.get("StorageDescriptor", {}).get("Location"):
+                    yield from self.storage_lineage.get_lineage_mcp(
+                        dataset_urn=dataset_urn,
+                        table=table,
+                        dataset_schema=schema_metadata,
+                    )
+
     def get_schema_names(self, inspector):
         assert isinstance(self.config, HiveConfig)
         # This condition restricts the ingestion to the specified database.
@@ -173,7 +771,7 @@ def get_schema_fields_for_column(
         pk_constraints: Optional[Dict[Any, Any]] = None,
         partition_keys: Optional[List[str]] = None,
         tags: Optional[List[str]] = None,
-    ) -> List[SchemaField]:
+    ) -> List[SchemaFieldClass]:
         fields = super().get_schema_fields_for_column(
             dataset_name,
             column,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py
index 21e7fad334331..d3941e7add0fd 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py
@@ -1,11 +1,17 @@
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union
 
-from datahub.emitter.mce_builder import make_data_flow_urn, make_data_job_urn
+from datahub.emitter.mce_builder import (
+    make_data_flow_urn,
+    make_data_job_urn,
+    make_data_platform_urn,
+    make_dataplatform_instance_urn,
+)
 from datahub.metadata.schema_classes import (
     DataFlowInfoClass,
     DataJobInfoClass,
     DataJobInputOutputClass,
+    DataPlatformInstanceClass,
 )
 
 
@@ -101,6 +107,7 @@ class StoredProcedure:
     flow: Union[MSSQLJob, MSSQLProceduresContainer]
     type: str = "STORED_PROCEDURE"
     source: str = "mssql"
+    code: Optional[str] = None
 
     @property
     def full_type(self) -> str:
@@ -203,6 +210,18 @@ def as_datajob_info_aspect(self) -> DataJobInfoClass:
             status=self.status,
         )
 
+    @property
+    def as_maybe_platform_instance_aspect(self) -> Optional[DataPlatformInstanceClass]:
+        if self.entity.flow.platform_instance:
+            return DataPlatformInstanceClass(
+                platform=make_data_platform_urn(self.entity.flow.orchestrator),
+                instance=make_dataplatform_instance_urn(
+                    platform=self.entity.flow.orchestrator,
+                    instance=self.entity.flow.platform_instance,
+                ),
+            )
+        return None
+
 
 @dataclass
 class MSSQLDataFlow:
@@ -237,3 +256,14 @@ def as_dataflow_info_aspect(self) -> DataFlowInfoClass:
             customProperties=self.flow_properties,
             externalUrl=self.external_url,
         )
+
+    @property
+    def as_maybe_platform_instance_aspect(self) -> Optional[DataPlatformInstanceClass]:
+        if self.entity.platform_instance:
+            return DataPlatformInstanceClass(
+                platform=make_data_platform_urn(self.entity.orchestrator),
+                instance=make_dataplatform_instance_urn(
+                    self.entity.orchestrator, self.entity.platform_instance
+                ),
+            )
+        return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
index c19b22a8622ca..9d8b67041998c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
@@ -5,8 +5,6 @@
 
 import pydantic
 import sqlalchemy.dialects.mssql
-
-# This import verifies that the dependencies are available.
 from pydantic.fields import Field
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.engine.base import Connection
@@ -24,6 +22,8 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.source import StructuredLogLevel
+from datahub.ingestion.api.source_helpers import auto_workunit
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.sql.mssql.job_models import (
     JobStep,
@@ -36,6 +36,9 @@
     ProcedureParameter,
     StoredProcedure,
 )
+from datahub.ingestion.source.sql.mssql.stored_procedure_lineage import (
+    generate_procedure_lineage,
+)
 from datahub.ingestion.source.sql.sql_common import (
     SQLAlchemySource,
     SqlWorkUnit,
@@ -45,12 +48,14 @@
     BasicSQLAlchemyConfig,
     make_sqlalchemy_uri,
 )
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.metadata.schema_classes import (
     BooleanTypeClass,
     NumberTypeClass,
     StringTypeClass,
     UnionTypeClass,
 )
+from datahub.utilities.file_backed_collections import FileBackedList
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -72,6 +77,11 @@ class SQLServerConfig(BasicSQLAlchemyConfig):
     include_stored_procedures_code: bool = Field(
         default=True, description="Include information about object code."
     )
+    procedure_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for stored procedures to filter in ingestion."
+        "Specify regex to match the entire procedure name in database.schema.procedure_name format. e.g. to match all procedures starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'",
+    )
     include_jobs: bool = Field(
         default=True,
         description="Include ingest of MSSQL Jobs. Requires access to the 'msdb' and 'sys' schema.",
@@ -99,6 +109,10 @@ class SQLServerConfig(BasicSQLAlchemyConfig):
         default=False,
         description="Enable to convert the SQL Server assets urns to lowercase",
     )
+    include_lineage: bool = Field(
+        default=True,
+        description="Enable lineage extraction for stored procedures",
+    )
 
     @pydantic.validator("uri_args")
     def passwords_match(cls, v, values, **kwargs):
@@ -154,6 +168,8 @@ class SQLServerSource(SQLAlchemySource):
     If you do use pyodbc, make sure to change the source type from `mssql` to `mssql-odbc` so that we pull in the right set of dependencies. This will be needed in most cases where encryption is required, such as managed SQL Server services in Azure.
     """
 
+    report: SQLSourceReport
+
     def __init__(self, config: SQLServerConfig, ctx: PipelineContext):
         super().__init__(config, ctx, "mssql")
         # Cache the table and column descriptions
@@ -161,6 +177,7 @@ def __init__(self, config: SQLServerConfig, ctx: PipelineContext):
         self.current_database = None
         self.table_descriptions: Dict[str, str] = {}
         self.column_descriptions: Dict[str, str] = {}
+        self.stored_procedures: FileBackedList[StoredProcedure] = FileBackedList()
         if self.config.include_descriptions:
             for inspector in self.get_inspectors():
                 db_name: str = self.get_db_name(inspector)
@@ -374,7 +391,7 @@ def loop_jobs(
     def loop_job_steps(
         self, job: MSSQLJob, job_steps: Dict[str, Any]
     ) -> Iterable[MetadataWorkUnit]:
-        for step_id, step_data in job_steps.items():
+        for _step_id, step_data in job_steps.items():
             step = JobStep(
                 job_name=job.formatted_name,
                 step_name=step_data["step_name"],
@@ -405,44 +422,57 @@ def loop_stored_procedures(  # noqa: C901
         data_flow = MSSQLDataFlow(entity=mssql_default_job)
         with inspector.engine.connect() as conn:
             procedures_data_list = self._get_stored_procedures(conn, db_name, schema)
-            procedures = [
-                StoredProcedure(flow=mssql_default_job, **procedure_data)
-                for procedure_data in procedures_data_list
-            ]
+            procedures: List[StoredProcedure] = []
+            for procedure_data in procedures_data_list:
+                procedure_full_name = f"{db_name}.{schema}.{procedure_data['name']}"
+                if not self.config.procedure_pattern.allowed(procedure_full_name):
+                    self.report.report_dropped(procedure_full_name)
+                    continue
+                procedures.append(
+                    StoredProcedure(flow=mssql_default_job, **procedure_data)
+                )
+
             if procedures:
                 yield from self.construct_flow_workunits(data_flow=data_flow)
             for procedure in procedures:
-                upstream = self._get_procedure_upstream(conn, procedure)
-                downstream = self._get_procedure_downstream(conn, procedure)
-                data_job = MSSQLDataJob(
-                    entity=procedure,
-                )
-                # TODO: because of this upstream and downstream are more dependencies,
-                #  can't be used as DataJobInputOutput.
-                #  Should be reorganized into lineage.
-                data_job.add_property("procedure_depends_on", str(upstream.as_property))
-                data_job.add_property(
-                    "depending_on_procedure", str(downstream.as_property)
-                )
-                procedure_definition, procedure_code = self._get_procedure_code(
-                    conn, procedure
-                )
-                if procedure_definition:
-                    data_job.add_property("definition", procedure_definition)
-                if sql_config.include_stored_procedures_code and procedure_code:
-                    data_job.add_property("code", procedure_code)
-                procedure_inputs = self._get_procedure_inputs(conn, procedure)
-                properties = self._get_procedure_properties(conn, procedure)
-                data_job.add_property(
-                    "input parameters", str([param.name for param in procedure_inputs])
-                )
-                for param in procedure_inputs:
-                    data_job.add_property(
-                        f"parameter {param.name}", str(param.properties)
-                    )
-                for property_name, property_value in properties.items():
-                    data_job.add_property(property_name, str(property_value))
-                yield from self.construct_job_workunits(data_job)
+                yield from self._process_stored_procedure(conn, procedure)
+
+    def _process_stored_procedure(
+        self, conn: Connection, procedure: StoredProcedure
+    ) -> Iterable[MetadataWorkUnit]:
+        upstream = self._get_procedure_upstream(conn, procedure)
+        downstream = self._get_procedure_downstream(conn, procedure)
+        data_job = MSSQLDataJob(
+            entity=procedure,
+        )
+        # TODO: because of this upstream and downstream are more dependencies,
+        #  can't be used as DataJobInputOutput.
+        #  Should be reorganized into lineage.
+        data_job.add_property("procedure_depends_on", str(upstream.as_property))
+        data_job.add_property("depending_on_procedure", str(downstream.as_property))
+        procedure_definition, procedure_code = self._get_procedure_code(conn, procedure)
+        procedure.code = procedure_code
+        if procedure_definition:
+            data_job.add_property("definition", procedure_definition)
+        if procedure_code and self.config.include_stored_procedures_code:
+            data_job.add_property("code", procedure_code)
+        procedure_inputs = self._get_procedure_inputs(conn, procedure)
+        properties = self._get_procedure_properties(conn, procedure)
+        data_job.add_property(
+            "input parameters", str([param.name for param in procedure_inputs])
+        )
+        for param in procedure_inputs:
+            data_job.add_property(f"parameter {param.name}", str(param.properties))
+        for property_name, property_value in properties.items():
+            data_job.add_property(property_name, str(property_value))
+        if self.config.include_lineage:
+            # These will be used to construct lineage
+            self.stored_procedures.append(procedure)
+        yield from self.construct_job_workunits(
+            data_job,
+            # For stored procedure lineage is ingested later
+            include_lineage=False,
+        )
 
     @staticmethod
     def _get_procedure_downstream(
@@ -546,8 +576,8 @@ def _get_procedure_code(
                 code_list.append(row["Text"])
                 if code_slice_text in re.sub(" +", " ", row["Text"].lower()).strip():
                     code_slice_index = index
-            definition = "\n".join(code_list[:code_slice_index])
-            code = "\n".join(code_list[code_slice_index:])
+            definition = "".join(code_list[:code_slice_index])
+            code = "".join(code_list[code_slice_index:])
         except ResourceClosedError:
             logger.warning(
                 "Connection was closed from procedure '%s'",
@@ -602,16 +632,25 @@ def _get_stored_procedures(
     def construct_job_workunits(
         self,
         data_job: MSSQLDataJob,
+        include_lineage: bool = True,
     ) -> Iterable[MetadataWorkUnit]:
         yield MetadataChangeProposalWrapper(
             entityUrn=data_job.urn,
             aspect=data_job.as_datajob_info_aspect,
         ).as_workunit()
 
-        yield MetadataChangeProposalWrapper(
-            entityUrn=data_job.urn,
-            aspect=data_job.as_datajob_input_output_aspect,
-        ).as_workunit()
+        data_platform_instance_aspect = data_job.as_maybe_platform_instance_aspect
+        if data_platform_instance_aspect:
+            yield MetadataChangeProposalWrapper(
+                entityUrn=data_job.urn,
+                aspect=data_platform_instance_aspect,
+            ).as_workunit()
+
+        if include_lineage:
+            yield MetadataChangeProposalWrapper(
+                entityUrn=data_job.urn,
+                aspect=data_job.as_datajob_input_output_aspect,
+            ).as_workunit()
         # TODO: Add SubType when it appear
 
     def construct_flow_workunits(
@@ -622,6 +661,13 @@ def construct_flow_workunits(
             entityUrn=data_flow.urn,
             aspect=data_flow.as_dataflow_info_aspect,
         ).as_workunit()
+
+        data_platform_instance_aspect = data_flow.as_maybe_platform_instance_aspect
+        if data_platform_instance_aspect:
+            yield MetadataChangeProposalWrapper(
+                entityUrn=data_flow.urn,
+                aspect=data_platform_instance_aspect,
+            ).as_workunit()
         # TODO: Add SubType when it appear
 
     def get_inspectors(self) -> Iterable[Inspector]:
@@ -664,3 +710,58 @@ def get_identifier(
             if self.config.convert_urns_to_lowercase
             else qualified_table_name
         )
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        yield from super().get_workunits_internal()
+
+        # This is done at the end so that we will have access to tables
+        # from all databases in schema_resolver and discovered_tables
+        for procedure in self.stored_procedures:
+            with self.report.report_exc(
+                message="Failed to parse stored procedure lineage",
+                context=procedure.full_name,
+                level=StructuredLogLevel.WARN,
+            ):
+                yield from auto_workunit(
+                    generate_procedure_lineage(
+                        schema_resolver=self.schema_resolver,
+                        procedure=procedure,
+                        procedure_job_urn=MSSQLDataJob(entity=procedure).urn,
+                        is_temp_table=self.is_temp_table,
+                    )
+                )
+
+    def is_temp_table(self, name: str) -> bool:
+        try:
+            parts = name.split(".")
+            table_name = parts[-1]
+            schema_name = parts[-2]
+            db_name = parts[-3]
+
+            if table_name.startswith("#"):
+                return True
+
+            # This is also a temp table if
+            #   1. this name would be allowed by the dataset patterns, and
+            #   2. we have a list of discovered tables, and
+            #   3. it's not in the discovered tables list
+            if (
+                self.config.database_pattern.allowed(db_name)
+                and self.config.schema_pattern.allowed(schema_name)
+                and self.config.table_pattern.allowed(name)
+                and self.standardize_identifier_case(name)
+                not in self.discovered_datasets
+            ):
+                logger.debug(f"inferred as temp table {name}")
+                return True
+
+        except Exception:
+            logger.warning(f"Error parsing table name {name} ")
+        return False
+
+    def standardize_identifier_case(self, table_ref_str: str) -> str:
+        return (
+            table_ref_str.lower()
+            if self.config.convert_urns_to_lowercase
+            else table_ref_str
+        )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py
new file mode 100644
index 0000000000000..b979a270a5528
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py
@@ -0,0 +1,84 @@
+import logging
+from typing import Callable, Iterable, Optional
+
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.source.sql.mssql.job_models import StoredProcedure
+from datahub.metadata.schema_classes import DataJobInputOutputClass
+from datahub.sql_parsing.datajob import to_datajob_input_output
+from datahub.sql_parsing.schema_resolver import SchemaResolver
+from datahub.sql_parsing.split_statements import split_statements
+from datahub.sql_parsing.sql_parsing_aggregator import (
+    ObservedQuery,
+    SqlParsingAggregator,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def parse_procedure_code(
+    *,
+    schema_resolver: SchemaResolver,
+    default_db: Optional[str],
+    default_schema: Optional[str],
+    code: str,
+    is_temp_table: Callable[[str], bool],
+    raise_: bool = False,
+) -> Optional[DataJobInputOutputClass]:
+    aggregator = SqlParsingAggregator(
+        platform=schema_resolver.platform,
+        env=schema_resolver.env,
+        schema_resolver=schema_resolver,
+        generate_lineage=True,
+        generate_queries=False,
+        generate_usage_statistics=False,
+        generate_operations=False,
+        generate_query_subject_fields=False,
+        generate_query_usage_statistics=False,
+        is_temp_table=is_temp_table,
+    )
+    for query in split_statements(code):
+        # TODO: We should take into account `USE x` statements.
+        aggregator.add_observed_query(
+            observed=ObservedQuery(
+                default_db=default_db,
+                default_schema=default_schema,
+                query=query,
+            )
+        )
+    if aggregator.report.num_observed_queries_failed and raise_:
+        logger.info(aggregator.report.as_string())
+        raise ValueError(
+            f"Failed to parse {aggregator.report.num_observed_queries_failed} queries."
+        )
+
+    mcps = list(aggregator.gen_metadata())
+    return to_datajob_input_output(
+        mcps=mcps,
+        ignore_extra_mcps=True,
+    )
+
+
+# Is procedure handling generic enough to be added to SqlParsingAggregator?
+def generate_procedure_lineage(
+    *,
+    schema_resolver: SchemaResolver,
+    procedure: StoredProcedure,
+    procedure_job_urn: str,
+    is_temp_table: Callable[[str], bool] = lambda _: False,
+    raise_: bool = False,
+) -> Iterable[MetadataChangeProposalWrapper]:
+    if procedure.code:
+        datajob_input_output = parse_procedure_code(
+            schema_resolver=schema_resolver,
+            default_db=procedure.db,
+            default_schema=procedure.schema,
+            code=procedure.code,
+            is_temp_table=is_temp_table,
+            raise_=raise_,
+        )
+
+        if datajob_input_output:
+            yield MetadataChangeProposalWrapper(
+                entityUrn=procedure_job_urn,
+                aspect=datajob_input_output,
+            )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py
index dcc1340c81d7b..52db3cd11a759 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py
@@ -1,11 +1,14 @@
+import datetime
 import logging
 import re
+
+# This import verifies that the dependencies are available.
+import sys
 from collections import defaultdict
 from typing import Any, Dict, Iterable, List, NoReturn, Optional, Tuple, Union, cast
 from unittest.mock import patch
 
-# This import verifies that the dependencies are available.
-import cx_Oracle
+import oracledb
 import pydantic
 import sqlalchemy.engine
 from pydantic.fields import Field
@@ -31,6 +34,9 @@
 
 logger = logging.getLogger(__name__)
 
+oracledb.version = "8.3.0"
+sys.modules["cx_Oracle"] = oracledb
+
 extra_oracle_types = {
     make_sqlalchemy_type("SDO_GEOMETRY"),
     make_sqlalchemy_type("SDO_POINT_TYPE"),
@@ -47,10 +53,10 @@ def _raise_err(exc: Exception) -> NoReturn:
 def output_type_handler(cursor, name, defaultType, size, precision, scale):
     """Add CLOB and BLOB support to Oracle connection."""
 
-    if defaultType == cx_Oracle.CLOB:
-        return cursor.var(cx_Oracle.LONG_STRING, arraysize=cursor.arraysize)
-    elif defaultType == cx_Oracle.BLOB:
-        return cursor.var(cx_Oracle.LONG_BINARY, arraysize=cursor.arraysize)
+    if defaultType == oracledb.CLOB:
+        return cursor.var(oracledb.DB_TYPE_LONG, arraysize=cursor.arraysize)
+    elif defaultType == oracledb.BLOB:
+        return cursor.var(oracledb.DB_TYPE_LONG_RAW, arraysize=cursor.arraysize)
 
 
 def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
@@ -58,9 +64,9 @@ def before_cursor_execute(conn, cursor, statement, parameters, context, executem
 
 
 class OracleConfig(BasicSQLAlchemyConfig):
-    # defaults
+    # TODO: Change scheme to oracle+oracledb when sqlalchemy>=2 is supported
     scheme: str = Field(
-        default="oracle+cx_oracle",
+        default="oracle",
         description="Will be set automatically to default value.",
     )
     service_name: Optional[str] = Field(
@@ -173,7 +179,6 @@ def get_table_names(self, schema: Optional[str] = None) -> List[str]:
         ]
 
     def get_view_names(self, schema: Optional[str] = None) -> List[str]:
-
         schema = self._inspector_instance.dialect.denormalize_name(
             schema or self.default_schema_name
         )
@@ -195,7 +200,6 @@ def get_view_names(self, schema: Optional[str] = None) -> List[str]:
     def get_columns(
         self, table_name: str, schema: Optional[str] = None, dblink: str = ""
     ) -> List[dict]:
-
         denormalized_table_name = self._inspector_instance.dialect.denormalize_name(
             table_name
         )
@@ -339,7 +343,6 @@ def get_columns(
         return columns
 
     def get_table_comment(self, table_name: str, schema: Optional[str] = None) -> Dict:
-
         denormalized_table_name = self._inspector_instance.dialect.denormalize_name(
             table_name
         )
@@ -411,7 +414,6 @@ def _get_constraint_data(
     def get_pk_constraint(
         self, table_name: str, schema: Optional[str] = None, dblink: str = ""
     ) -> Dict:
-
         denormalized_table_name = self._inspector_instance.dialect.denormalize_name(
             table_name
         )
@@ -453,7 +455,6 @@ def get_pk_constraint(
     def get_foreign_keys(
         self, table_name: str, schema: Optional[str] = None, dblink: str = ""
     ) -> List:
-
         denormalized_table_name = self._inspector_instance.dialect.denormalize_name(
             table_name
         )
@@ -535,7 +536,6 @@ def fkey_rec():
     def get_view_definition(
         self, view_name: str, schema: Optional[str] = None
     ) -> Union[str, None]:
-
         denormalized_view_name = self._inspector_instance.dialect.denormalize_name(
             view_name
         )
@@ -632,3 +632,52 @@ def get_workunits(self):
             clear=False,
         ):
             return super().get_workunits()
+
+    def generate_profile_candidates(
+        self,
+        inspector: Inspector,
+        threshold_time: Optional[datetime.datetime],
+        schema: str,
+    ) -> Optional[List[str]]:
+        tables_table_name = (
+            "ALL_TABLES" if self.config.data_dictionary_mode == "ALL" else "DBA_TABLES"
+        )
+
+        # If stats are available , they are used even if they are stale.
+        # Assuming that the table would typically grow over time, this will ensure to filter
+        # large tables known at stats collection time from profiling candidates.
+        # If stats are not available (NULL), such tables are not filtered and are considered
+        # as profiling candidates.
+        cursor = inspector.bind.execute(
+            sql.text(
+                f"""SELECT
+                            t.OWNER,
+                            t.TABLE_NAME,
+                            t.NUM_ROWS,
+                            t.LAST_ANALYZED,
+                            COALESCE(t.NUM_ROWS * t.AVG_ROW_LEN, 0) / (1024 * 1024 * 1024) AS SIZE_GB
+                        FROM {tables_table_name} t
+                        WHERE t.OWNER = :owner
+                        AND (t.NUM_ROWS < :table_row_limit OR t.NUM_ROWS IS NULL)
+                        AND COALESCE(t.NUM_ROWS * t.AVG_ROW_LEN, 0) / (1024 * 1024 * 1024) < :table_size_limit
+                """
+            ),
+            dict(
+                owner=inspector.dialect.denormalize_name(schema),
+                table_row_limit=self.config.profiling.profile_table_row_limit,
+                table_size_limit=self.config.profiling.profile_table_size_limit,
+            ),
+        )
+
+        TABLE_NAME_COL_LOC = 1
+        return [
+            self.get_identifier(
+                schema=schema,
+                entity=inspector.dialect.normalize_name(row[TABLE_NAME_COL_LOC])
+                or _raise_err(
+                    ValueError(f"Invalid table name: {row[TABLE_NAME_COL_LOC]}")
+                ),
+                inspector=inspector,
+            )
+            for row in cursor
+        ]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
index 238fd88f1c950..4e22930e7a2a0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
@@ -32,6 +32,7 @@
     make_data_platform_urn,
     make_dataplatform_instance_urn,
     make_dataset_urn_with_platform_instance,
+    make_schema_field_urn,
     make_tag_urn,
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -50,7 +51,6 @@
 from datahub.ingestion.glossary.classification_mixin import (
     SAMPLE_SIZE_MULTIPLIER,
     ClassificationHandler,
-    ClassificationReportMixin,
 )
 from datahub.ingestion.source.common.data_reader import DataReader
 from datahub.ingestion.source.common.subtypes import (
@@ -58,6 +58,7 @@
     DatasetSubTypes,
 )
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source.sql.sql_utils import (
     add_table_to_schema_container,
     downgrade_schema_from_v2,
@@ -73,7 +74,6 @@
 )
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
-    StaleEntityRemovalSourceReport,
 )
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionSourceBase,
@@ -117,9 +117,7 @@
 )
 from datahub.telemetry import telemetry
 from datahub.utilities.file_backed_collections import FileBackedDict
-from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.registries.domain_registry import DomainRegistry
-from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport
 from datahub.utilities.sqlalchemy_type_converter import (
     get_native_data_type_for_sqlalchemy_type,
 )
@@ -133,43 +131,6 @@
 logger: logging.Logger = logging.getLogger(__name__)
 
 
-@dataclass
-class SQLSourceReport(StaleEntityRemovalSourceReport, ClassificationReportMixin):
-    tables_scanned: int = 0
-    views_scanned: int = 0
-    entities_profiled: int = 0
-    filtered: LossyList[str] = field(default_factory=LossyList)
-
-    query_combiner: Optional[SQLAlchemyQueryCombinerReport] = None
-
-    num_view_definitions_parsed: int = 0
-    num_view_definitions_failed_parsing: int = 0
-    num_view_definitions_failed_column_parsing: int = 0
-    view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList)
-
-    def report_entity_scanned(self, name: str, ent_type: str = "table") -> None:
-        """
-        Entity could be a view or a table
-        """
-        if ent_type == "table":
-            self.tables_scanned += 1
-        elif ent_type == "view":
-            self.views_scanned += 1
-        else:
-            raise KeyError(f"Unknown entity {ent_type}.")
-
-    def report_entity_profiled(self, name: str) -> None:
-        self.entities_profiled += 1
-
-    def report_dropped(self, ent_name: str) -> None:
-        self.filtered.append(ent_name)
-
-    def report_from_query_combiner(
-        self, query_combiner_report: SQLAlchemyQueryCombinerReport
-    ) -> None:
-        self.query_combiner = query_combiner_report
-
-
 class SqlWorkUnit(MetadataWorkUnit):
     pass
 
@@ -351,7 +312,7 @@ class SQLAlchemySource(StatefulIngestionSourceBase, TestableSource):
 
     def __init__(self, config: SQLCommonConfig, ctx: PipelineContext, platform: str):
         super().__init__(config, ctx)
-        self.config = config
+        self.config: SQLCommonConfig = config
         self.platform = platform
         self.report: SQLSourceReport = SQLSourceReport()
         self.profile_metadata_info: ProfileMetadata = ProfileMetadata()
@@ -391,6 +352,7 @@ def __init__(self, config: SQLCommonConfig, ctx: PipelineContext, platform: str)
             platform_instance=self.config.platform_instance,
             env=self.config.env,
         )
+        self.discovered_datasets: Set[str] = set()
         self._view_definition_cache: MutableMapping[str, str]
         if self.config.use_file_backed_cache:
             self._view_definition_cache = FileBackedDict[str]()
@@ -620,6 +582,8 @@ def get_view_lineage(self) -> Iterable[MetadataWorkUnit]:
             generate_operations=False,
         )
         for dataset_name in self._view_definition_cache.keys():
+            # TODO: Ensure that the lineage generated from the view definition
+            # matches the dataset_name.
             view_definition = self._view_definition_cache[dataset_name]
             result = self._run_sql_parser(
                 dataset_name,
@@ -669,7 +633,7 @@ def get_foreign_key_metadata(
         )
 
         source_fields = [
-            f"urn:li:schemaField:({dataset_urn},{f})"
+            make_schema_field_urn(dataset_urn, f)
             for f in fk_dict["constrained_columns"]
         ]
         foreign_dataset = make_dataset_urn_with_platform_instance(
@@ -679,7 +643,7 @@ def get_foreign_key_metadata(
             env=self.config.env,
         )
         foreign_fields = [
-            f"urn:li:schemaField:({foreign_dataset},{f})"
+            make_schema_field_urn(foreign_dataset, f)
             for f in fk_dict["referred_columns"]
         ]
 
@@ -830,8 +794,9 @@ def _process_table(
         self._classify(dataset_name, schema, table, data_reader, schema_metadata)
 
         dataset_snapshot.aspects.append(schema_metadata)
-        if self.config.include_view_lineage:
+        if self._save_schema_to_resolver():
             self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata)
+            self.discovered_datasets.add(dataset_name)
         db_name = self.get_db_name(inspector)
 
         yield from self.add_table_to_schema_container(
@@ -1096,6 +1061,20 @@ def loop_views(
                 exc=e,
             )
 
+    def _get_view_definition(self, inspector: Inspector, schema: str, view: str) -> str:
+        try:
+            view_definition = inspector.get_view_definition(view, schema)
+            if view_definition is None:
+                view_definition = ""
+            else:
+                # Some dialects return a TextClause instead of a raw string,
+                # so we need to convert them to a string.
+                view_definition = str(view_definition)
+        except NotImplementedError:
+            view_definition = ""
+
+        return view_definition
+
     def _process_view(
         self,
         dataset_name: str,
@@ -1114,7 +1093,10 @@ def _process_view(
             columns = inspector.get_columns(view, schema)
         except KeyError:
             # For certain types of views, we are unable to fetch the list of columns.
-            self.warn(logger, dataset_name, "unable to get schema for this view")
+            self.report.warning(
+                message="Unable to get schema for a view",
+                context=f"{dataset_name}",
+            )
             schema_metadata = None
         else:
             schema_fields = self.get_schema_fields(dataset_name, columns, inspector)
@@ -1125,21 +1107,15 @@ def _process_view(
                 columns,
                 canonical_schema=schema_fields,
             )
-            if self.config.include_view_lineage:
+            if self._save_schema_to_resolver():
                 self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata)
+                self.discovered_datasets.add(dataset_name)
+
         description, properties, _ = self.get_table_properties(inspector, schema, view)
-        try:
-            view_definition = inspector.get_view_definition(view, schema)
-            if view_definition is None:
-                view_definition = ""
-            else:
-                # Some dialects return a TextClause instead of a raw string,
-                # so we need to convert them to a string.
-                view_definition = str(view_definition)
-        except NotImplementedError:
-            view_definition = ""
-        properties["view_definition"] = view_definition
         properties["is_view"] = "True"
+
+        view_definition = self._get_view_definition(inspector, schema, view)
+        properties["view_definition"] = view_definition
         if view_definition and self.config.include_view_lineage:
             self._view_definition_cache[dataset_name] = view_definition
 
@@ -1171,15 +1147,14 @@ def _process_view(
             entityUrn=dataset_urn,
             aspect=SubTypesClass(typeNames=[DatasetSubTypes.VIEW]),
         ).as_workunit()
-        if "view_definition" in properties:
-            view_definition_string = properties["view_definition"]
-            view_properties_aspect = ViewPropertiesClass(
-                materialized=False, viewLanguage="SQL", viewLogic=view_definition_string
-            )
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=view_properties_aspect,
-            ).as_workunit()
+
+        view_properties_aspect = ViewPropertiesClass(
+            materialized=False, viewLanguage="SQL", viewLogic=view_definition
+        )
+        yield MetadataChangeProposalWrapper(
+            entityUrn=dataset_urn,
+            aspect=view_properties_aspect,
+        ).as_workunit()
 
         if self.config.domain and self.domain_registry:
             yield from get_domain_wu(
@@ -1189,6 +1164,11 @@ def _process_view(
                 domain_registry=self.domain_registry,
             )
 
+    def _save_schema_to_resolver(self):
+        return self.config.include_view_lineage or (
+            hasattr(self.config, "include_lineage") and self.config.include_lineage
+        )
+
     def _run_sql_parser(
         self, view_identifier: str, query: str, schema_resolver: SchemaResolver
     ) -> Optional[SqlParsingResult]:
@@ -1228,6 +1208,8 @@ def _run_sql_parser(
             )
         else:
             self.report.num_view_definitions_parsed += 1
+            if raw_lineage.out_tables != [view_urn]:
+                self.report.num_view_definitions_view_urn_mismatch += 1
         return view_definition_lineage_helper(raw_lineage, view_urn)
 
     def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]:
@@ -1273,17 +1255,22 @@ def generate_profile_candidates(
     def is_dataset_eligible_for_profiling(
         self,
         dataset_name: str,
-        sql_config: SQLCommonConfig,
+        schema: str,
         inspector: Inspector,
         profile_candidates: Optional[List[str]],
     ) -> bool:
-        return (
-            sql_config.table_pattern.allowed(dataset_name)
-            and sql_config.profile_pattern.allowed(dataset_name)
-        ) and (
-            profile_candidates is None
-            or (profile_candidates is not None and dataset_name in profile_candidates)
-        )
+        if not (
+            self.config.table_pattern.allowed(dataset_name)
+            and self.config.profile_pattern.allowed(dataset_name)
+        ):
+            self.report.profiling_skipped_table_profile_pattern[schema] += 1
+            return False
+
+        if profile_candidates is not None and dataset_name not in profile_candidates:
+            self.report.profiling_skipped_other[schema] += 1
+            return False
+
+        return True
 
     def loop_profiler_requests(
         self,
@@ -1298,7 +1285,7 @@ def loop_profiler_requests(
         if (
             sql_config.profiling.profile_if_updated_since_days is not None
             or sql_config.profiling.profile_table_size_limit is not None
-            or sql_config.profiling.profile_table_row_limit is None
+            or sql_config.profiling.profile_table_row_limit is not None
         ):
             try:
                 threshold_time: Optional[datetime.datetime] = None
@@ -1319,8 +1306,9 @@ def loop_profiler_requests(
                 schema=schema, entity=table, inspector=inspector
             )
             if not self.is_dataset_eligible_for_profiling(
-                dataset_name, sql_config, inspector, profile_candidates
+                dataset_name, schema, inspector, profile_candidates
             ):
+                self.report.num_tables_not_eligible_profiling[schema] += 1
                 if self.config.profiling.report_dropped_profiles:
                     self.report.report_dropped(f"profile of {dataset_name}")
                 continue
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
index 3ead59eed2d39..7d82d99412ffe 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
@@ -2,6 +2,8 @@
 from abc import abstractmethod
 from typing import Any, Dict, Optional
 
+import cachetools
+import cachetools.keys
 import pydantic
 from pydantic import Field
 from sqlalchemy.engine import URL
@@ -27,6 +29,7 @@
     StatefulIngestionConfigBase,
 )
 from datahub.ingestion.source_config.operation_config import is_profiling_enabled
+from datahub.utilities.cachetools_keys import self_methodkey
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -115,6 +118,13 @@ class SQLCommonConfig(
     # Custom Stateful Ingestion settings
     stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
 
+    # TRICKY: The operation_config is time-dependent. Because we don't want to change
+    # whether or not we're running profiling mid-ingestion, we cache the result of this method.
+    # TODO: This decorator should be moved to the is_profiling_enabled(operation_config) method.
+    @cachetools.cached(
+        cache=cachetools.LRUCache(maxsize=1),
+        key=self_methodkey,
+    )
     def is_profiling_enabled(self) -> bool:
         return self.profiling.enabled and is_profiling_enabled(
             self.profiling.operation_config
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py
index 9c8e475e7b307..bd6c23cc2d464 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py
@@ -1,6 +1,6 @@
 import logging
 from abc import abstractmethod
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from typing import Dict, Iterable, List, Optional, Union, cast
 
@@ -14,42 +14,13 @@
     DatahubGEProfiler,
     GEProfilerRequest,
 )
-from datahub.ingestion.source.sql.sql_common import SQLSourceReport
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
 from datahub.ingestion.source.sql.sql_generic import BaseTable, BaseView
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source.sql.sql_utils import check_table_with_profile_pattern
 from datahub.ingestion.source.state.profiling_state_handler import ProfilingHandler
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import DatasetProfile
 from datahub.metadata.com.linkedin.pegasus2avro.timeseries import PartitionType
-from datahub.utilities.stats_collections import TopKDict, int_top_k_dict
-
-
-@dataclass
-class DetailedProfilerReportMixin:
-    profiling_skipped_not_updated: TopKDict[str, int] = field(
-        default_factory=int_top_k_dict
-    )
-    profiling_skipped_size_limit: TopKDict[str, int] = field(
-        default_factory=int_top_k_dict
-    )
-
-    profiling_skipped_row_limit: TopKDict[str, int] = field(
-        default_factory=int_top_k_dict
-    )
-
-    profiling_skipped_table_profile_pattern: TopKDict[str, int] = field(
-        default_factory=int_top_k_dict
-    )
-
-    profiling_skipped_other: TopKDict[str, int] = field(default_factory=int_top_k_dict)
-
-    num_tables_not_eligible_profiling: Dict[str, int] = field(
-        default_factory=int_top_k_dict
-    )
-
-
-class ProfilingSqlReport(DetailedProfilerReportMixin, SQLSourceReport):
-    pass
 
 
 @dataclass
@@ -65,7 +36,7 @@ class GenericProfiler:
     def __init__(
         self,
         config: SQLCommonConfig,
-        report: ProfilingSqlReport,
+        report: SQLSourceReport,
         platform: str,
         state_handler: Optional[ProfilingHandler] = None,
     ) -> None:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py
new file mode 100644
index 0000000000000..c445ce44a9144
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py
@@ -0,0 +1,76 @@
+from dataclasses import dataclass, field
+from typing import Dict, Optional
+
+from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalSourceReport,
+)
+from datahub.utilities.lossy_collections import LossyList
+from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport
+from datahub.utilities.stats_collections import TopKDict, int_top_k_dict
+
+
+@dataclass
+class DetailedProfilerReportMixin:
+    profiling_skipped_not_updated: TopKDict[str, int] = field(
+        default_factory=int_top_k_dict
+    )
+    profiling_skipped_size_limit: TopKDict[str, int] = field(
+        default_factory=int_top_k_dict
+    )
+
+    profiling_skipped_row_limit: TopKDict[str, int] = field(
+        default_factory=int_top_k_dict
+    )
+
+    profiling_skipped_table_profile_pattern: TopKDict[str, int] = field(
+        default_factory=int_top_k_dict
+    )
+
+    profiling_skipped_other: TopKDict[str, int] = field(default_factory=int_top_k_dict)
+
+    num_tables_not_eligible_profiling: Dict[str, int] = field(
+        default_factory=int_top_k_dict
+    )
+
+
+@dataclass
+class SQLSourceReport(
+    StaleEntityRemovalSourceReport,
+    ClassificationReportMixin,
+    DetailedProfilerReportMixin,
+):
+    tables_scanned: int = 0
+    views_scanned: int = 0
+    entities_profiled: int = 0
+    filtered: LossyList[str] = field(default_factory=LossyList)
+
+    query_combiner: Optional[SQLAlchemyQueryCombinerReport] = None
+
+    num_view_definitions_parsed: int = 0
+    num_view_definitions_view_urn_mismatch: int = 0
+    num_view_definitions_failed_parsing: int = 0
+    num_view_definitions_failed_column_parsing: int = 0
+    view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList)
+
+    def report_entity_scanned(self, name: str, ent_type: str = "table") -> None:
+        """
+        Entity could be a view or a table
+        """
+        if ent_type == "table":
+            self.tables_scanned += 1
+        elif ent_type == "view":
+            self.views_scanned += 1
+        else:
+            raise KeyError(f"Unknown entity {ent_type}.")
+
+    def report_entity_profiled(self, name: str) -> None:
+        self.entities_profiled += 1
+
+    def report_dropped(self, ent_name: str) -> None:
+        self.filtered.append(ent_name)
+
+    def report_from_query_combiner(
+        self, query_combiner_report: SQLAlchemyQueryCombinerReport
+    ) -> None:
+        self.query_combiner = query_combiner_report
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
index 8ea4209784063..9ec73a9af96dc 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
@@ -1,5 +1,5 @@
 import re
-from typing import Any, Dict, ValuesView
+from typing import Any, Dict, Optional, Type, Union, ValuesView
 
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     ArrayType,
@@ -16,14 +16,28 @@
     UnionType,
 )
 
-# these can be obtained by running `select format_type(oid, null),* from pg_type;`
-# we've omitted the types without a meaningful DataHub type (e.g. postgres-specific types, index vectors, etc.)
-# (run `\copy (select format_type(oid, null),* from pg_type) to 'pg_type.csv' csv header;` to get a CSV)
+DATAHUB_FIELD_TYPE = Union[
+    ArrayType,
+    BooleanType,
+    BytesType,
+    DateType,
+    EnumType,
+    MapType,
+    NullType,
+    NumberType,
+    RecordType,
+    StringType,
+    TimeType,
+    UnionType,
+]
 
-# we map from format_type since this is what dbt uses
-# see https://github.com/fishtown-analytics/dbt/blob/master/plugins/postgres/dbt/include/postgres/macros/catalog.sql#L22
 
-# see https://www.npgsql.org/dev/types.html for helpful type annotations
+# These can be obtained by running `select format_type(oid, null),* from pg_type;`
+# We've omitted the types without a meaningful DataHub type (e.g. postgres-specific types, index vectors, etc.)
+# (run `\copy (select format_type(oid, null),* from pg_type) to 'pg_type.csv' csv header;` to get a CSV)
+# We map from format_type since this is what dbt uses.
+# See https://github.com/fishtown-analytics/dbt/blob/master/plugins/postgres/dbt/include/postgres/macros/catalog.sql#L22
+# See https://www.npgsql.org/dev/types.html for helpful type annotations
 POSTGRES_TYPES_MAP: Dict[str, Any] = {
     "boolean": BooleanType,
     "bytea": BytesType,
@@ -262,7 +276,6 @@ def resolve_vertica_modified_type(type_string: str) -> Any:
     return VERTICA_SQL_TYPES_MAP[type_string]
 
 
-# see https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html
 SNOWFLAKE_TYPES_MAP: Dict[str, Any] = {
     "NUMBER": NumberType,
     "DECIMAL": NumberType,
@@ -298,6 +311,18 @@ def resolve_vertica_modified_type(type_string: str) -> Any:
     "GEOGRAPHY": None,
 }
 
+
+def resolve_snowflake_modified_type(type_string: str) -> Any:
+    # Match types with precision and scale, e.g., 'DECIMAL(38,0)'
+    match = re.match(r"([a-zA-Z_]+)\(\d+,\s\d+\)", type_string)
+    if match:
+        modified_type_base = match.group(1)  # Extract the base type
+        return SNOWFLAKE_TYPES_MAP.get(modified_type_base, None)
+
+    # Fallback for types without precision/scale
+    return SNOWFLAKE_TYPES_MAP.get(type_string, None)
+
+
 # see https://github.com/googleapis/python-bigquery-sqlalchemy/blob/main/sqlalchemy_bigquery/_types.py#L32
 BIGQUERY_TYPES_MAP: Dict[str, Any] = {
     "STRING": StringType,
@@ -366,6 +391,7 @@ def resolve_vertica_modified_type(type_string: str) -> Any:
     "row": RecordType,
     "map": MapType,
     "array": ArrayType,
+    "json": RecordType,
 }
 
 # https://docs.aws.amazon.com/athena/latest/ug/data-types.html
@@ -430,3 +456,54 @@ def resolve_vertica_modified_type(type_string: str) -> Any:
     "geography": None,
     "uuid": StringType,
 }
+
+
+_merged_mapping = {
+    "boolean": BooleanType,
+    "date": DateType,
+    "time": TimeType,
+    "numeric": NumberType,
+    "text": StringType,
+    "timestamp with time zone": DateType,
+    "timestamp without time zone": DateType,
+    "integer": NumberType,
+    "float8": NumberType,
+    "struct": RecordType,
+    **POSTGRES_TYPES_MAP,
+    **SNOWFLAKE_TYPES_MAP,
+    **BIGQUERY_TYPES_MAP,
+    **SPARK_SQL_TYPES_MAP,
+    **TRINO_SQL_TYPES_MAP,
+    **ATHENA_SQL_TYPES_MAP,
+    **VERTICA_SQL_TYPES_MAP,
+}
+
+
+def resolve_sql_type(
+    column_type: Optional[str],
+    platform: Optional[str] = None,
+) -> Optional[DATAHUB_FIELD_TYPE]:
+    # In theory, we should use the platform-specific mapping where available.
+    # However, the types don't ever conflict, so the merged mapping is fine.
+    TypeClass: Optional[Type[DATAHUB_FIELD_TYPE]] = (
+        _merged_mapping.get(column_type) if column_type else None
+    )
+
+    if TypeClass is None and column_type:
+        # resolve a modified type
+        if platform == "trino":
+            TypeClass = resolve_trino_modified_type(column_type)
+        elif platform == "athena":
+            TypeClass = resolve_athena_modified_type(column_type)
+        elif platform == "postgres" or platform == "redshift":
+            # Redshift uses a variant of Postgres, so we can use the same logic.
+            TypeClass = resolve_postgres_modified_type(column_type)
+        elif platform == "vertica":
+            TypeClass = resolve_vertica_modified_type(column_type)
+        elif platform == "snowflake":
+            # Snowflake types are uppercase, so we check that.
+            TypeClass = resolve_snowflake_modified_type(column_type.upper())
+
+    if TypeClass:
+        return TypeClass()
+    return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
index 53b1ddfcde595..e42564975c3d1 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -44,7 +44,7 @@
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.source.sql.sql_common import SqlWorkUnit, register_custom_type
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
-from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source.sql.two_tier_sql_source import (
     TwoTierSQLAlchemyConfig,
     TwoTierSQLAlchemySource,
@@ -330,7 +330,7 @@ def optimized_get_view_definition(
 
 
 @dataclass
-class TeradataReport(ProfilingSqlReport, IngestionStageReport, BaseTimeWindowReport):
+class TeradataReport(SQLSourceReport, IngestionStageReport, BaseTimeWindowReport):
     num_queries_parsed: int = 0
     num_view_ddl_parsed: int = 0
     num_table_parse_failures: int = 0
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
index a340f049731c4..92487d48b99e6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
@@ -27,7 +27,6 @@
 from datahub.ingestion.source.common.data_reader import DataReader
 from datahub.ingestion.source.sql.sql_common import (
     SQLAlchemySource,
-    SQLSourceReport,
     SqlWorkUnit,
     get_schema_metadata,
 )
@@ -35,6 +34,7 @@
     BasicSQLAlchemyConfig,
     SQLCommonConfig,
 )
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source.sql.sql_utils import get_domain_wu
 from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage
@@ -536,7 +536,7 @@ def loop_profiler_requests(
             )
 
             if not self.is_dataset_eligible_for_profiling(
-                dataset_name, sql_config, inspector, profile_candidates
+                dataset_name, schema, inspector, profile_candidates
             ):
                 if self.config.profiling.report_dropped_profiles:
                     self.report.report_dropped(f"profile of {dataset_name}")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py
index f011aa7bdd19e..2b10ca1fa57ed 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py
@@ -8,6 +8,11 @@
 from datahub.utilities.dedup_list import deduplicate_list
 from datahub.utilities.urns.urn import guess_entity_type
 
+STATEFUL_INGESTION_IGNORED_ENTITY_TYPES = {
+    "dataProcessInstance",
+    "query",
+}
+
 
 def pydantic_state_migrator(mapping: Dict[str, str]) -> classmethod:
     # mapping would be something like:
@@ -127,8 +132,11 @@ def get_percent_entities_changed(
         :param old_checkpoint_state: the old checkpoint state to compute the relative change percent against.
         :return: (1-|intersection(self, old_checkpoint_state)| / |old_checkpoint_state|) * 100.0
         """
+
+        old_urns_filtered = filter_ignored_entity_types(old_checkpoint_state.urns)
+
         return compute_percent_entities_changed(
-            new_entities=self.urns, old_entities=old_checkpoint_state.urns
+            new_entities=self.urns, old_entities=old_urns_filtered
         )
 
     def urn_count(self) -> int:
@@ -138,7 +146,11 @@ def urn_count(self) -> int:
 def compute_percent_entities_changed(
     new_entities: List[str], old_entities: List[str]
 ) -> float:
-    (overlap_count, old_count, _,) = _get_entity_overlap_and_cardinalities(
+    (
+        overlap_count,
+        old_count,
+        _,
+    ) = _get_entity_overlap_and_cardinalities(
         new_entities=new_entities, old_entities=old_entities
     )
 
@@ -153,3 +165,19 @@ def _get_entity_overlap_and_cardinalities(
     new_set = set(new_entities)
     old_set = set(old_entities)
     return len(new_set.intersection(old_set)), len(old_set), len(new_set)
+
+
+def filter_ignored_entity_types(urns: List[str]) -> List[str]:
+    # We previously stored ignored entity urns (e.g.dataProcessInstance) in state.
+    # For smoother transition from old checkpoint state, without requiring explicit
+    # setting of `fail_safe_threshold` due to removal of irrelevant urns from new state,
+    # here, we would ignore irrelevant urns from percentage entities changed computation
+    # This special handling can be removed after few months.
+    return [
+        urn
+        for urn in urns
+        if not any(
+            urn.startswith(f"urn:li:{entityType}")
+            for entityType in STATEFUL_INGESTION_IGNORED_ENTITY_TYPES
+        )
+    ]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/redundant_run_skip_handler.py b/metadata-ingestion/src/datahub/ingestion/source/state/redundant_run_skip_handler.py
index a2e078f233f1d..8630a959d3f6a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state/redundant_run_skip_handler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state/redundant_run_skip_handler.py
@@ -69,7 +69,7 @@ def _init_job_id(self) -> JobId:
         platform: Optional[str] = None
         source_class = type(self.source)
         if hasattr(source_class, "get_platform_name"):
-            platform = source_class.get_platform_name()  # type: ignore
+            platform = source_class.get_platform_name()
 
         # Default name for everything else
         job_name_suffix = self.get_job_name_suffix()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py
index 9d77e13a0f3c2..d4fcbf09924e9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py
@@ -11,7 +11,10 @@
 from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import JobId
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.state.checkpoint import Checkpoint
-from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState
+from datahub.ingestion.source.state.entity_removal_state import (
+    STATEFUL_INGESTION_IGNORED_ENTITY_TYPES,
+    GenericCheckpointState,
+)
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfig,
     StatefulIngestionConfigBase,
@@ -27,11 +30,6 @@
 
 logger: logging.Logger = logging.getLogger(__name__)
 
-STATEFUL_INGESTION_IGNORED_ENTITY_TYPES = {
-    "dataProcessInstance",
-    "query",
-}
-
 
 class StatefulStaleMetadataRemovalConfig(StatefulIngestionConfig):
     """
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py b/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py
index 442abb3aaf4cf..8f4a53ffc3ed5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py
@@ -17,7 +17,7 @@
 
 
 class DatahubIngestionStateProviderConfig(IngestionCheckpointingProviderConfig):
-    datahub_api: DatahubClientConfig = DatahubClientConfig()
+    datahub_api: Optional[DatahubClientConfig] = None
 
 
 class DatahubIngestionCheckpointingProvider(IngestionCheckpointingProviderBase):
@@ -31,8 +31,8 @@ def __init__(
         self.graph = graph
         if not self._is_server_stateful_ingestion_capable():
             raise ConfigurationError(
-                "Datahub server is not capable of supporting stateful ingestion."
-                " Please consider upgrading to the latest server version to use this feature."
+                "Datahub server is not capable of supporting stateful ingestion. "
+                "Please consider upgrading to the latest server version to use this feature."
             )
 
     @classmethod
@@ -40,11 +40,15 @@ def create(
         cls, config_dict: Dict[str, Any], ctx: PipelineContext
     ) -> "DatahubIngestionCheckpointingProvider":
         config = DatahubIngestionStateProviderConfig.parse_obj(config_dict)
-        if ctx.graph:
-            # Use the pipeline-level graph if set
+        if config.datahub_api is not None:
+            return cls(DataHubGraph(config.datahub_api))
+        elif ctx.graph:
+            # Use the pipeline-level graph if set.
             return cls(ctx.graph)
         else:
-            return cls(DataHubGraph(config.datahub_api))
+            raise ValueError(
+                "A graph instance is required. Either pass one in the pipeline context, or set it explicitly in the stateful ingestion provider config."
+            )
 
     def _is_server_stateful_ingestion_capable(self) -> bool:
         server_config = self.graph.get_config() if self.graph else None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py
index 858281f880359..1da233bf0b22a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/superset.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py
@@ -1,10 +1,12 @@
 import json
 import logging
+from datetime import datetime
 from functools import lru_cache
-from typing import Dict, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Optional
 
 import dateutil.parser as dp
 import requests
+from pydantic import BaseModel
 from pydantic.class_validators import root_validator, validator
 from pydantic.fields import Field
 
@@ -16,7 +18,9 @@
 from datahub.emitter.mce_builder import (
     make_chart_urn,
     make_dashboard_urn,
+    make_data_platform_urn,
     make_dataset_urn,
+    make_dataset_urn_with_platform_instance,
     make_domain_urn,
 )
 from datahub.emitter.mcp_builder import add_domain_to_entity_wu
@@ -31,6 +35,7 @@
 )
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.sql.sql_types import resolve_sql_type
 from datahub.ingestion.source.sql.sqlalchemy_uri_mapper import (
     get_platform_from_sqlalchemy_uri,
 )
@@ -47,16 +52,26 @@
     AuditStamp,
     ChangeAuditStamps,
     Status,
+    TimeStamp,
 )
 from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import (
     ChartSnapshot,
     DashboardSnapshot,
+    DatasetSnapshot,
 )
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
+from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+    MySqlDDL,
+    NullType,
+    SchemaField,
+    SchemaFieldDataType,
+    SchemaMetadata,
+)
 from datahub.metadata.schema_classes import (
     ChartInfoClass,
     ChartTypeClass,
     DashboardInfoClass,
+    DatasetPropertiesClass,
 )
 from datahub.utilities import config_clean
 from datahub.utilities.registries.domain_registry import DomainRegistry
@@ -83,6 +98,28 @@
 }
 
 
+platform_without_databases = ["druid"]
+
+
+class SupersetDataset(BaseModel):
+    id: int
+    table_name: str
+    changed_on_utc: Optional[str] = None
+    explore_url: Optional[str] = ""
+
+    @property
+    def modified_dt(self) -> Optional[datetime]:
+        if self.changed_on_utc:
+            return dp.parse(self.changed_on_utc)
+        return None
+
+    @property
+    def modified_ts(self) -> Optional[int]:
+        if self.modified_dt:
+            return int(self.modified_dt.timestamp() * 1000)
+        return None
+
+
 class SupersetConfig(
     StatefulIngestionConfigBase, EnvConfigMixin, PlatformInstanceConfigMixin
 ):
@@ -101,15 +138,17 @@ class SupersetConfig(
     )
     username: Optional[str] = Field(default=None, description="Superset username.")
     password: Optional[str] = Field(default=None, description="Superset password.")
-    api_key: Optional[str] = Field(default=None, description="Preset.io API key.")
-    api_secret: Optional[str] = Field(default=None, description="Preset.io API secret.")
-    manager_uri: str = Field(
-        default="https://api.app.preset.io", description="Preset.io API URL"
-    )
     # Configuration for stateful ingestion
     stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field(
         default=None, description="Superset Stateful Ingestion Config."
     )
+    ingest_dashboards: bool = Field(
+        default=True, description="Enable to ingest dashboards."
+    )
+    ingest_charts: bool = Field(default=True, description="Enable to ingest charts.")
+    ingest_datasets: bool = Field(
+        default=False, description="Enable to ingest datasets."
+    )
 
     provider: str = Field(default="db", description="Superset provider.")
     options: Dict = Field(default={}, description="")
@@ -121,6 +160,10 @@ class SupersetConfig(
         description="Can be used to change mapping for database names in superset to what you have in datahub",
     )
 
+    class Config:
+        # This is required to allow preset configs to get parsed
+        extra = "allow"
+
     @validator("connect_uri", "display_uri")
     def remove_trailing_slash(cls, v):
         return config_clean.remove_trailing_slashes(v)
@@ -227,6 +270,28 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source:
         config = SupersetConfig.parse_obj(config_dict)
         return cls(ctx, config)
 
+    def paginate_entity_api_results(self, entity_type, page_size=100):
+        current_page = 0
+        total_items = page_size
+
+        while current_page * page_size < total_items:
+            response = self.session.get(
+                f"{self.config.connect_uri}/api/v1/{entity_type}/",
+                params={"q": f"(page:{current_page},page_size:{page_size})"},
+            )
+
+            if response.status_code != 200:
+                logger.warning(f"Failed to get {entity_type} data: {response.text}")
+
+            payload = response.json()
+            # Update total_items with the actual count from the response
+            total_items = payload.get("count", total_items)
+            # Yield each item in the result, this gets passed into the construct functions
+            for item in payload.get("result", []):
+                yield item
+
+            current_page += 1
+
     @lru_cache(maxsize=None)
     def get_platform_from_database_id(self, database_id):
         database_response = self.session.get(
@@ -243,32 +308,56 @@ def get_platform_from_database_id(self, database_id):
             return "athena"
         if platform_name == "clickhousedb":
             return "clickhouse"
+        if platform_name == "postgresql":
+            return "postgres"
         return platform_name
 
     @lru_cache(maxsize=None)
-    def get_datasource_urn_from_id(self, datasource_id):
+    def get_dataset_info(self, dataset_id: int) -> dict:
         dataset_response = self.session.get(
-            f"{self.config.connect_uri}/api/v1/dataset/{datasource_id}"
-        ).json()
+            f"{self.config.connect_uri}/api/v1/dataset/{dataset_id}",
+        )
+        if dataset_response.status_code != 200:
+            logger.warning(f"Failed to get dataset info: {dataset_response.text}")
+            dataset_response.raise_for_status()
+        return dataset_response.json()
+
+    def get_datasource_urn_from_id(
+        self, dataset_response: dict, platform_instance: str
+    ) -> str:
         schema_name = dataset_response.get("result", {}).get("schema")
         table_name = dataset_response.get("result", {}).get("table_name")
         database_id = dataset_response.get("result", {}).get("database", {}).get("id")
+        platform = self.get_platform_from_database_id(database_id)
+
         database_name = (
             dataset_response.get("result", {}).get("database", {}).get("database_name")
         )
         database_name = self.config.database_alias.get(database_name, database_name)
 
+        # Druid do not have a database concept and has a limited schema concept, but they are nonetheless reported
+        # from superset. There is only one database per platform instance, and one schema named druid, so it would be
+        # redundant to systemically store them both in the URN.
+        if platform in platform_without_databases:
+            database_name = None
+
+        if platform == "druid" and schema_name == "druid":
+            # Follow DataHub's druid source convention.
+            schema_name = None
+
         if database_id and table_name:
             return make_dataset_urn(
-                platform=self.get_platform_from_database_id(database_id),
+                platform=platform,
                 name=".".join(
                     name for name in [database_name, schema_name, table_name] if name
                 ),
                 env=self.config.env,
             )
-        return None
+        raise ValueError("Could not construct dataset URN")
 
-    def construct_dashboard_from_api_data(self, dashboard_data):
+    def construct_dashboard_from_api_data(
+        self, dashboard_data: dict
+    ) -> DashboardSnapshot:
         dashboard_urn = make_dashboard_urn(
             platform=self.platform,
             name=dashboard_data["id"],
@@ -323,7 +412,7 @@ def construct_dashboard_from_api_data(self, dashboard_data):
         }
 
         if dashboard_data.get("certified_by"):
-            custom_properties["CertifiedBy"] = dashboard_data.get("certified_by")
+            custom_properties["CertifiedBy"] = dashboard_data.get("certified_by", "")
             custom_properties["CertificationDetails"] = str(
                 dashboard_data.get("certification_details")
             )
@@ -341,38 +430,25 @@ def construct_dashboard_from_api_data(self, dashboard_data):
         return dashboard_snapshot
 
     def emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]:
-        current_dashboard_page = 0
-        # we will set total dashboards to the actual number after we get the response
-        total_dashboards = PAGE_SIZE
-
-        while current_dashboard_page * PAGE_SIZE <= total_dashboards:
-            dashboard_response = self.session.get(
-                f"{self.config.connect_uri}/api/v1/dashboard/",
-                params=f"q=(page:{current_dashboard_page},page_size:{PAGE_SIZE})",
-            )
-            if dashboard_response.status_code != 200:
-                logger.warning(
-                    f"Failed to get dashboard data: {dashboard_response.text}"
-                )
-            dashboard_response.raise_for_status()
-
-            payload = dashboard_response.json()
-            total_dashboards = payload.get("count") or 0
-
-            current_dashboard_page += 1
-
-            for dashboard_data in payload["result"]:
+        for dashboard_data in self.paginate_entity_api_results("dashboard", PAGE_SIZE):
+            try:
                 dashboard_snapshot = self.construct_dashboard_from_api_data(
                     dashboard_data
                 )
-                mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot)
-                yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce)
-                yield from self._get_domain_wu(
-                    title=dashboard_data.get("dashboard_title", ""),
-                    entity_urn=dashboard_snapshot.urn,
+            except Exception as e:
+                self.report.warning(
+                    f"Failed to construct dashboard snapshot. Dashboard name: {dashboard_data.get('dashboard_title')}. Error: \n{e}"
                 )
+                continue
+            # Emit the dashboard
+            mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot)
+            yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce)
+            yield from self._get_domain_wu(
+                title=dashboard_data.get("dashboard_title", ""),
+                entity_urn=dashboard_snapshot.urn,
+            )
 
-    def construct_chart_from_chart_data(self, chart_data):
+    def construct_chart_from_chart_data(self, chart_data: dict) -> ChartSnapshot:
         chart_urn = make_chart_urn(
             platform=self.platform,
             name=chart_data["id"],
@@ -398,9 +474,12 @@ def construct_chart_from_chart_data(self, chart_data):
         chart_url = f"{self.config.display_uri}{chart_data.get('url', '')}"
 
         datasource_id = chart_data.get("datasource_id")
-        datasource_urn = self.get_datasource_urn_from_id(datasource_id)
+        dataset_response = self.get_dataset_info(datasource_id)
+        datasource_urn = self.get_datasource_urn_from_id(
+            dataset_response, self.platform
+        )
 
-        params = json.loads(chart_data.get("params"))
+        params = json.loads(chart_data.get("params", "{}"))
         metrics = [
             get_metric_name(metric)
             for metric in (params.get("metrics", []) or [params.get("metric")])
@@ -450,36 +529,124 @@ def construct_chart_from_chart_data(self, chart_data):
         return chart_snapshot
 
     def emit_chart_mces(self) -> Iterable[MetadataWorkUnit]:
-        current_chart_page = 0
-        # we will set total charts to the actual number after we get the response
-        total_charts = PAGE_SIZE
-
-        while current_chart_page * PAGE_SIZE <= total_charts:
-            chart_response = self.session.get(
-                f"{self.config.connect_uri}/api/v1/chart/",
-                params=f"q=(page:{current_chart_page},page_size:{PAGE_SIZE})",
+        for chart_data in self.paginate_entity_api_results("chart", PAGE_SIZE):
+            try:
+                chart_snapshot = self.construct_chart_from_chart_data(chart_data)
+
+                mce = MetadataChangeEvent(proposedSnapshot=chart_snapshot)
+            except Exception as e:
+                self.report.warning(
+                    f"Failed to construct chart snapshot. Chart name: {chart_data.get('table_name')}. Error: \n{e}"
+                )
+                continue
+            # Emit the chart
+            yield MetadataWorkUnit(id=chart_snapshot.urn, mce=mce)
+            yield from self._get_domain_wu(
+                title=chart_data.get("slice_name", ""),
+                entity_urn=chart_snapshot.urn,
             )
-            if chart_response.status_code != 200:
-                logger.warning(f"Failed to get chart data: {chart_response.text}")
-            chart_response.raise_for_status()
 
-            current_chart_page += 1
+    def gen_schema_fields(self, column_data: List[Dict[str, str]]) -> List[SchemaField]:
+        schema_fields: List[SchemaField] = []
+        for col in column_data:
+            col_type = (col.get("type") or "").lower()
+            data_type = resolve_sql_type(col_type)
+            if data_type is None:
+                data_type = NullType()
+
+            field = SchemaField(
+                fieldPath=col.get("column_name", ""),
+                type=SchemaFieldDataType(data_type),
+                nativeDataType="",
+                description=col.get("column_name", ""),
+                nullable=True,
+            )
+            schema_fields.append(field)
+        return schema_fields
+
+    def gen_schema_metadata(
+        self,
+        dataset_response: dict,
+    ) -> SchemaMetadata:
+        dataset_response = dataset_response.get("result", {})
+        column_data = dataset_response.get("columns", [])
+        schema_metadata = SchemaMetadata(
+            schemaName=dataset_response.get("table_name", ""),
+            platform=make_data_platform_urn(self.platform),
+            version=0,
+            hash="",
+            platformSchema=MySqlDDL(tableSchema=""),
+            fields=self.gen_schema_fields(column_data),
+        )
+        return schema_metadata
 
-            payload = chart_response.json()
-            total_charts = payload["count"]
-            for chart_data in payload["result"]:
-                chart_snapshot = self.construct_chart_from_chart_data(chart_data)
+    def gen_dataset_urn(self, datahub_dataset_name: str) -> str:
+        return make_dataset_urn_with_platform_instance(
+            platform=self.platform,
+            name=datahub_dataset_name,
+            platform_instance=self.config.platform_instance,
+            env=self.config.env,
+        )
 
-                mce = MetadataChangeEvent(proposedSnapshot=chart_snapshot)
-                yield MetadataWorkUnit(id=chart_snapshot.urn, mce=mce)
-                yield from self._get_domain_wu(
-                    title=chart_data.get("slice_name", ""),
-                    entity_urn=chart_snapshot.urn,
+    def construct_dataset_from_dataset_data(
+        self, dataset_data: dict
+    ) -> DatasetSnapshot:
+        dataset_response = self.get_dataset_info(dataset_data.get("id"))
+        dataset = SupersetDataset(**dataset_response["result"])
+        datasource_urn = self.get_datasource_urn_from_id(
+            dataset_response, self.platform
+        )
+
+        dataset_url = f"{self.config.display_uri}{dataset.explore_url or ''}"
+
+        dataset_info = DatasetPropertiesClass(
+            name=dataset.table_name,
+            description="",
+            lastModified=TimeStamp(time=dataset.modified_ts)
+            if dataset.modified_ts
+            else None,
+            externalUrl=dataset_url,
+        )
+        aspects_items: List[Any] = []
+        aspects_items.extend(
+            [
+                self.gen_schema_metadata(dataset_response),
+                dataset_info,
+            ]
+        )
+
+        dataset_snapshot = DatasetSnapshot(
+            urn=datasource_urn,
+            aspects=aspects_items,
+        )
+        return dataset_snapshot
+
+    def emit_dataset_mces(self) -> Iterable[MetadataWorkUnit]:
+        for dataset_data in self.paginate_entity_api_results("dataset", PAGE_SIZE):
+            try:
+                dataset_snapshot = self.construct_dataset_from_dataset_data(
+                    dataset_data
                 )
+                mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
+            except Exception as e:
+                self.report.warning(
+                    f"Failed to construct dataset snapshot. Dataset name: {dataset_data.get('table_name')}. Error: \n{e}"
+                )
+                continue
+            # Emit the dataset
+            yield MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce)
+            yield from self._get_domain_wu(
+                title=dataset_data.get("table_name", ""),
+                entity_urn=dataset_snapshot.urn,
+            )
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
-        yield from self.emit_dashboard_mces()
-        yield from self.emit_chart_mces()
+        if self.config.ingest_dashboards:
+            yield from self.emit_dashboard_mces()
+        if self.config.ingest_charts:
+            yield from self.emit_chart_mces()
+        if self.config.ingest_datasets:
+            yield from self.emit_dataset_mces()
 
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py
index 9f011790990ec..2b7aac2bea1d0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py
@@ -1,3 +1,4 @@
+import json
 import logging
 import re
 import time
@@ -26,13 +27,18 @@
 from pydantic.fields import Field
 from requests.adapters import HTTPAdapter
 from tableauserverclient import (
+    GroupItem,
+    PermissionsRule,
     PersonalAccessTokenAuth,
     Server,
     ServerResponseError,
     SiteItem,
     TableauAuth,
 )
-from tableauserverclient.server.endpoint.exceptions import NonXMLResponseError
+from tableauserverclient.server.endpoint.exceptions import (
+    InternalServerError,
+    NonXMLResponseError,
+)
 from urllib3 import Retry
 
 import datahub.emitter.mce_builder as builder
@@ -46,6 +52,7 @@
     DatasetSourceConfigMixin,
 )
 from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
+from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import (
     ContainerKey,
@@ -65,6 +72,7 @@
     CapabilityReport,
     MetadataWorkUnitProcessor,
     Source,
+    StructuredLogLevel,
     TestableSource,
     TestConnectionReport,
 )
@@ -107,6 +115,8 @@
     tableau_field_to_schema_field,
     workbook_graphql_query,
 )
+from datahub.ingestion.source.tableau.tableau_server_wrapper import UserInfo
+from datahub.ingestion.source.tableau.tableau_validation import check_user_role
 from datahub.metadata.com.linkedin.pegasus2avro.common import (
     AuditStamp,
     ChangeAuditStamps,
@@ -163,7 +173,7 @@
 
 try:
     # On earlier versions of the tableauserverclient, the NonXMLResponseError
-    # was thrown when reauthentication was needed. We'll keep both exceptions
+    # was thrown when reauthentication was necessary. We'll keep both exceptions
     # around for now, but can remove this in the future.
     from tableauserverclient.server.endpoint.exceptions import (  # type: ignore
         NotSignedInError,
@@ -216,6 +226,11 @@ class TableauConnectionConfig(ConfigModel):
         description="Whether to verify SSL certificates. If using self-signed certificates, set to false or provide the path to the .pem certificate bundle.",
     )
 
+    session_trust_env: bool = Field(
+        False,
+        description="Configures the trust_env property in the requests session. If set to false (default value) it will bypass proxy settings. See https://requests.readthedocs.io/en/latest/api/#requests.Session.trust_env for more information.",
+    )
+
     extract_column_level_lineage: bool = Field(
         True,
         description="When enabled, extracts column-level lineage from Tableau Datasources",
@@ -265,8 +280,7 @@ def make_tableau_client(self, site: str) -> Server:
                 },
             )
 
-            # From https://stackoverflow.com/a/50159273/5004662.
-            server._session.trust_env = False
+            server._session.trust_env = self.session_trust_env
 
             # Setup request retries.
             adapter = HTTPAdapter(
@@ -282,22 +296,35 @@ def make_tableau_client(self, site: str) -> Server:
             server.auth.sign_in(authentication)
             return server
         except ServerResponseError as e:
+            message = f"Unable to login (invalid/expired credentials or missing permissions): {str(e)}"
             if isinstance(authentication, PersonalAccessTokenAuth):
                 # Docs on token expiry in Tableau:
                 # https://help.tableau.com/current/server/en-us/security_personal_access_tokens.htm#token-expiry
-                logger.info(
-                    "Error authenticating with Tableau. Note that Tableau personal access tokens "
-                    "expire if not used for 15 days or if over 1 year old"
-                )
-            raise ValueError(
-                f"Unable to login (invalid/expired credentials or missing permissions): {str(e)}"
-            ) from e
+                message = f"Error authenticating with Tableau. Note that Tableau personal access tokens expire if not used for 15 days or if over 1 year old: {str(e)}"
+            raise ValueError(message) from e
         except Exception as e:
             raise ValueError(
                 f"Unable to login (check your Tableau connection and credentials): {str(e)}"
             ) from e
 
 
+class PermissionIngestionConfig(ConfigModel):
+    enable_workbooks: bool = Field(
+        default=True,
+        description="Whether or not to enable group permission ingestion for workbooks. "
+        "Default: True",
+    )
+
+    group_name_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Filter for Tableau group names when ingesting group permissions. "
+        "For example, you could filter for groups that include the term 'Consumer' in their name by adding '^.*Consumer$' to the allow list."
+        "By default, all groups will be ingested. "
+        "You can both allow and deny groups based on their name using their name, or a Regex pattern. "
+        "Deny patterns always take precedence over allow patterns. ",
+    )
+
+
 class TableauConfig(
     DatasetLineageProviderConfigBase,
     StatefulIngestionConfigBase,
@@ -332,7 +359,7 @@ class TableauConfig(
 
     project_path_separator: str = Field(
         default="/",
-        description="The separator used for the project_pattern field between project names. By default, we use a slash. "
+        description="The separator used for the project_path_pattern field between project names. By default, we use a slash. "
         "You can change this if your Tableau projects contain slashes in their names, and you'd like to filter by project.",
     )
 
@@ -357,11 +384,6 @@ class TableauConfig(
         description="[advanced] Number of metadata objects (e.g. CustomSQLTable, PublishedDatasource, etc) to query at a time using the Tableau API.",
     )
 
-    fetch_size: int = Field(
-        default=250,
-        description="Specifies the number of records to retrieve in each batch during a query execution.",
-    )
-
     # We've found that even with a small workbook page size (e.g. 10), the Tableau API often
     # returns warnings like this:
     # {
@@ -459,6 +481,27 @@ class TableauConfig(
         description="When enabled, sites are added as containers and therefore visible in the folder structure within Datahub.",
     )
 
+    permission_ingestion: Optional[PermissionIngestionConfig] = Field(
+        default=None,
+        description="Configuration settings for ingesting Tableau groups and their capabilities as custom properties.",
+    )
+
+    ingest_hidden_assets: bool = Field(
+        True,
+        description="When enabled, hidden views and dashboards are ingested into Datahub. "
+        "If a dashboard or view is hidden in Tableau the luid is blank. Default of this config field is True.",
+    )
+
+    tags_for_hidden_assets: List[str] = Field(
+        default=[],
+        description="Tags to be added to hidden dashboards and views. If a dashboard or view is hidden in Tableau the luid is blank. "
+        "This can only be used with ingest_tags enabled as it will overwrite tags entered from the UI.",
+    )
+
+    _fetch_size = pydantic_removed_field(
+        "fetch_size",
+    )
+
     # pre = True because we want to take some decision before pydantic initialize the configuration to default values
     @root_validator(pre=True)
     def projects_backward_compatibility(cls, values: Dict) -> Dict:
@@ -484,6 +527,20 @@ def projects_backward_compatibility(cls, values: Dict) -> Dict:
 
         return values
 
+    @root_validator()
+    def validate_config_values(cls, values: Dict) -> Dict:
+        tags_for_hidden_assets = values.get("tags_for_hidden_assets")
+        ingest_tags = values.get("ingest_tags")
+        if (
+            not ingest_tags
+            and tags_for_hidden_assets
+            and len(tags_for_hidden_assets) > 0
+        ):
+            raise ValueError(
+                "tags_for_hidden_assets is only allowed with ingest_tags enabled. Be aware that this will overwrite tags entered from the UI."
+            )
+        return values
+
 
 class WorkbookKey(ContainerKey):
     workbook_id: str
@@ -570,7 +627,43 @@ class TableauSourceReport(StaleEntityRemovalSourceReport):
     num_datasource_field_skipped_no_name: int = 0
     num_csql_field_skipped_no_name: int = 0
     num_table_field_skipped_no_name: int = 0
+    # lineage
+    num_tables_with_upstream_lineage: int = 0
+    num_upstream_table_lineage: int = 0
+    num_upstream_fine_grained_lineage: int = 0
     num_upstream_table_skipped_no_name: int = 0
+    num_upstream_table_skipped_no_columns: int = 0
+    num_upstream_table_failed_generate_reference: int = 0
+    num_upstream_table_lineage_failed_parse_sql: int = 0
+    num_upstream_fine_grained_lineage_failed_parse_sql: int = 0
+    num_hidden_assets_skipped: int = 0
+    logged_in_user: List[UserInfo] = []
+
+
+def report_user_role(report: TableauSourceReport, server: Server) -> None:
+    title: str = "Insufficient Permissions"
+    message: str = "The user must have the `Site Administrator Explorer` role to perform metadata ingestion."
+    try:
+        # TableauSiteSource instance is per site, so each time we need to find-out user detail
+        # the site-role might be different on another site
+        logged_in_user: UserInfo = UserInfo.from_server(server=server)
+
+        if not logged_in_user.has_site_administrator_explorer_privileges():
+            report.warning(
+                title=title,
+                message=message,
+                context=f"user-name={logged_in_user.user_name}, role={logged_in_user.site_role}, site_id={logged_in_user.site_id}",
+            )
+
+        report.logged_in_user.append(logged_in_user)
+
+    except Exception as e:
+        report.warning(
+            title=title,
+            message="Failed to verify the user's role. The user must have `Site Administrator Explorer` role.",
+            context=f"{e}",
+            exc=e,
+        )
 
 
 @platform_name("Tableau")
@@ -615,6 +708,7 @@ def _authenticate(self, site_content_url: str) -> None:
         try:
             logger.info(f"Authenticated to Tableau site: '{site_content_url}'")
             self.server = self.config.make_tableau_client(site_content_url)
+            report_user_role(report=self.report, server=self.server)
         # Note that we're not catching ConfigurationError, since we want that to throw.
         except ValueError as e:
             self.report.failure(
@@ -628,9 +722,17 @@ def test_connection(config_dict: dict) -> TestConnectionReport:
         test_report = TestConnectionReport()
         try:
             source_config = TableauConfig.parse_obj_allow_extras(config_dict)
-            source_config.make_tableau_client(source_config.site)
+
+            server = source_config.make_tableau_client(source_config.site)
+
             test_report.basic_connectivity = CapabilityReport(capable=True)
+
+            test_report.capability_report = check_user_role(
+                logged_in_user=UserInfo.from_server(server=server)
+            )
+
         except Exception as e:
+            logger.warning(f"{e}", exc_info=e)
             test_report.basic_connectivity = CapabilityReport(
                 capable=False, failure_reason=str(e)
             )
@@ -671,6 +773,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                         config=self.config,
                         ctx=self.ctx,
                         site=site,
+                        site_id=site.id,
                         report=self.report,
                         server=self.server,
                         platform=self.platform,
@@ -678,11 +781,19 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     logger.info(f"Ingesting assets of site '{site.content_url}'.")
                     yield from site_source.ingest_tableau_site()
             else:
-                site = self.server.sites.get_by_id(self.server.site_id)
+                site = None
+                with self.report.report_exc(
+                    title="Unable to fetch site details. Site hierarchy may be incomplete and external urls may be missing.",
+                    message="This usually indicates missing permissions. Ensure that you have all necessary permissions.",
+                    level=StructuredLogLevel.WARN,
+                ):
+                    site = self.server.sites.get_by_id(self.server.site_id)
+
                 site_source = TableauSiteSource(
                     config=self.config,
                     ctx=self.ctx,
                     site=site,
+                    site_id=self.server.site_id,
                     report=self.report,
                     server=self.server,
                     platform=self.platform,
@@ -693,6 +804,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 title="Failed to Retrieve Tableau Metadata",
                 message="Unable to retrieve metadata from tableau.",
                 context=str(md_exception),
+                exc=md_exception,
             )
 
     def close(self) -> None:
@@ -714,7 +826,8 @@ def __init__(
         self,
         config: TableauConfig,
         ctx: PipelineContext,
-        site: SiteItem,
+        site: Optional[SiteItem],
+        site_id: Optional[str],
         report: TableauSourceReport,
         server: Server,
         platform: str,
@@ -723,15 +836,24 @@ def __init__(
         self.report = report
         self.server: Server = server
         self.ctx: PipelineContext = ctx
-        self.site: SiteItem = site
         self.platform = platform
 
+        self.site: Optional[SiteItem] = site
+        if site_id is not None:
+            self.site_id: str = site_id
+        else:
+            assert self.site is not None, "site or site_id is required"
+            assert self.site.id is not None, "site_id is required when site is provided"
+            self.site_id = self.site.id
+
         self.database_tables: Dict[str, DatabaseTable] = {}
         self.tableau_stat_registry: Dict[str, UsageStat] = {}
         self.tableau_project_registry: Dict[str, TableauProject] = {}
         self.workbook_project_map: Dict[str, str] = {}
         self.datasource_project_map: Dict[str, str] = {}
 
+        self.group_map: Dict[str, GroupItem] = {}
+
         # This map keeps track of the database server connection hostnames.
         self.database_server_hostname_map: Dict[str, str] = {}
         # This list keeps track of sheets in workbooks so that we retrieve those
@@ -750,6 +872,8 @@ def __init__(
         # when emitting custom SQL data sources.
         self.custom_sql_ids_being_used: List[str] = []
 
+        report_user_role(report=report, server=server)
+
     @property
     def no_env_browse_prefix(self) -> str:
         # Prefix to use with browse path (v1)
@@ -775,10 +899,9 @@ def dataset_browse_prefix(self) -> str:
         return f"/{self.config.env.lower()}{self.no_env_browse_prefix}"
 
     def _re_authenticate(self):
-        tableau_auth: Union[
-            TableauAuth, PersonalAccessTokenAuth
-        ] = self.config.get_tableau_auth(self.site.content_url)
-        self.server.auth.sign_in(tableau_auth)
+        # Sign-in again may not be enough because Tableau sometimes caches invalid sessions
+        # so we need to recreate the Tableau Server object
+        self.server = self.config.make_tableau_client(self.site_id)
 
     @property
     def site_content_url(self) -> Optional[str]:
@@ -795,6 +918,7 @@ def _populate_usage_stat_registry(self) -> None:
             if not view.id:
                 continue
             self.tableau_stat_registry[view.id] = UsageStat(view_count=view.total_views)
+        logger.info(f"Got Tableau stats for {len(self.tableau_stat_registry)} assets")
         logger.debug("Tableau stats %s", self.tableau_stat_registry)
 
     def _populate_database_server_hostname_map(self) -> None:
@@ -845,7 +969,7 @@ def form_path(project_id: str) -> List[str]:
                 ancestors = [cur_proj.name]
                 while cur_proj.parent_id is not None:
                     if cur_proj.parent_id not in all_project_map:
-                        self.report.report_warning(
+                        self.report.warning(
                             "project-issue",
                             f"Parent project {cur_proj.parent_id} not found. We need Site Administrator Explorer permissions.",
                         )
@@ -877,19 +1001,36 @@ def _is_allowed_project(self, project: TableauProject) -> bool:
         return is_allowed
 
     def _is_denied_project(self, project: TableauProject) -> bool:
-        # Either project name or project path should exist in deny
-        for deny_pattern in self.config.project_pattern.deny:
-            # Either name or project path is denied
-            if re.match(
-                deny_pattern, project.name, self.config.project_pattern.regex_flags
-            ) or re.match(
-                deny_pattern,
-                self._get_project_path(project),
-                self.config.project_pattern.regex_flags,
-            ):
-                return True
-        logger.info(f"project({project.name}) is not denied as per project_pattern")
-        return False
+        """
+        Why use an explicit denial check instead of the `AllowDenyPattern.allowed` method?
+
+        Consider a scenario where a Tableau site contains four projects: A, B, C, and D, with the following hierarchical relationship:
+
+        - **A**
+          - **B** (Child of A)
+          - **C** (Child of A)
+        - **D**
+
+        In this setup:
+
+        - `project_pattern` is configured with `allow: ["A"]` and `deny: ["B"]`.
+        - `extract_project_hierarchy` is set to `True`.
+
+        The goal is to extract assets from project A and its children while explicitly denying the child project B.
+
+        If we rely solely on the `project_pattern.allowed()` method, project C's assets will not be ingested.
+        This happens because project C is not explicitly included in the `allow` list, nor is it part of the `deny` list.
+        However, since `extract_project_hierarchy` is enabled, project C should ideally be included in the ingestion process unless explicitly denied.
+
+        To address this, the function explicitly checks the deny regex to ensure that project C’s assets are ingested if it is not specifically denied in the deny list. This approach ensures that the hierarchy is respected while adhering to the configured allow/deny rules.
+        """
+
+        # Either project_pattern or project_path_pattern is set in a recipe
+        # TableauConfig.projects_backward_compatibility ensures that at least one of these properties is configured.
+
+        return self.config.project_pattern.denied(
+            project.name
+        ) or self.config.project_path_pattern.denied(self._get_project_path(project))
 
     def _init_tableau_project_registry(self, all_project_map: dict) -> None:
         list_of_skip_projects: List[TableauProject] = []
@@ -917,9 +1058,11 @@ def _init_tableau_project_registry(self, all_project_map: dict) -> None:
             for project in list_of_skip_projects:
                 if (
                     project.parent_id in projects_to_ingest
-                    and self._is_denied_project(project) is False
+                    and not self._is_denied_project(project)
                 ):
-                    logger.debug(f"Project {project.name} is added in project registry")
+                    logger.debug(
+                        f"Project {project.name} is added in project registry as it's a child project and not explicitly denied in `deny` list"
+                    )
                     projects_to_ingest[project.id] = project
 
         # We rely on automatic browse paths (v2) when creating containers. That's why we need to sort the projects here.
@@ -943,8 +1086,11 @@ def _init_datasource_registry(self) -> None:
                 self.datasource_project_map[ds.id] = ds.project_id
         except Exception as e:
             self.report.get_all_datasources_query_failed = True
-            logger.info(f"Get all datasources query failed due to error {e}")
-            logger.debug("Error stack trace", exc_info=True)
+            self.report.warning(
+                title="Unexpected Query Error",
+                message="Get all datasources query failed due to error",
+                exc=e,
+            )
 
     def _init_workbook_registry(self) -> None:
         if self.server is None:
@@ -993,13 +1139,18 @@ def get_data_platform_instance(self) -> DataPlatformInstanceClass:
             ),
         )
 
+    def _is_hidden_view(self, dashboard_or_view: Dict) -> bool:
+        # LUID is blank if the view is hidden in the workbook.
+        # More info here: https://help.tableau.com/current/api/metadata_api/en-us/reference/view.doc.html
+        return not dashboard_or_view.get(c.LUID)
+
     def get_connection_object_page(
         self,
         query: str,
         connection_type: str,
         query_filter: str,
         current_cursor: Optional[str],
-        fetch_size: int = 250,
+        fetch_size: int,
         retry_on_auth_error: bool = True,
         retries_remaining: Optional[int] = None,
     ) -> Tuple[dict, Optional[str], int]:
@@ -1048,6 +1199,24 @@ def get_connection_object_page(
                 retry_on_auth_error=False,
                 retries_remaining=retries_remaining - 1,
             )
+
+        except InternalServerError as ise:
+            # In some cases Tableau Server returns 504 error, which is a timeout error, so it worths to retry.
+            if ise.code == 504:
+                if retries_remaining <= 0:
+                    raise ise
+                return self.get_connection_object_page(
+                    query=query,
+                    connection_type=connection_type,
+                    query_filter=query_filter,
+                    fetch_size=fetch_size,
+                    current_cursor=current_cursor,
+                    retry_on_auth_error=False,
+                    retries_remaining=retries_remaining - 1,
+                )
+            else:
+                raise ise
+
         except OSError:
             # In tableauseverclient 0.26 (which was yanked and released in 0.28 on 2023-10-04),
             # the request logic was changed to use threads.
@@ -1110,7 +1279,6 @@ def get_connection_object_page(
                     )
 
                 if node_limit_errors:
-                    logger.debug(f"Node Limit Error. query_data {query_data}")
                     self.report.warning(
                         title="Tableau Data Exceed Predefined Limit",
                         message="The numbers of record in result set exceeds a predefined limit. Increase the tableau "
@@ -1182,7 +1350,6 @@ def get_connection_objects(
         page_size = page_size_override or self.config.page_size
 
         filter_pages = get_filter_pages(query_filter, page_size)
-
         for filter_page in filter_pages:
             has_next_page = 1
             current_cursor: Optional[str] = None
@@ -1198,7 +1365,11 @@ def get_connection_objects(
                     connection_type=connection_type,
                     query_filter=filter_,
                     current_cursor=current_cursor,
-                    fetch_size=self.config.fetch_size,
+                    # `filter_page` contains metadata object IDs (e.g., Project IDs, Field IDs, Sheet IDs, etc.).
+                    # The number of IDs is always less than or equal to page_size.
+                    # If the IDs are primary keys, the number of metadata objects to load matches the number of records to return.
+                    # In our case, mostly, the IDs are primary key, therefore, fetch_size is set equal to page_size.
+                    fetch_size=page_size,
                 )
 
                 yield from connection_objects.get(c.NODES) or []
@@ -1226,9 +1397,10 @@ def emit_workbooks(self) -> Iterable[MetadataWorkUnit]:
                     wrk_id: Optional[str] = workbook.get(c.ID)
                     prj_name: Optional[str] = workbook.get(c.PROJECT_NAME)
 
-                    logger.debug(
-                        f"Skipping workbook {wrk_name}({wrk_id}) as it is project {prj_name}({project_luid}) not "
-                        f"present in project registry"
+                    self.report.warning(
+                        title="Skipping Missing Workbook",
+                        message="Skipping workbook as its project is not present in project registry",
+                        context=f"workbook={wrk_name}({wrk_id}), project={prj_name}({project_luid})",
                     )
                     continue
 
@@ -1261,7 +1433,7 @@ def _create_upstream_table_lineage(
         datasource: dict,
         browse_path: Optional[str],
         is_embedded_ds: bool = False,
-    ) -> Tuple:
+    ) -> Tuple[List[Upstream], List[FineGrainedLineage]]:
         upstream_tables: List[Upstream] = []
         fine_grained_lineages: List[FineGrainedLineage] = []
         table_id_to_urn = {}
@@ -1422,7 +1594,8 @@ def get_upstream_tables(
                 c.COLUMNS_CONNECTION
             ].get("totalCount")
             if not is_custom_sql and not num_tbl_cols:
-                logger.debug(
+                self.report.num_upstream_table_skipped_no_columns += 1
+                logger.warning(
                     f"Skipping upstream table with id {table[c.ID]}, no columns: {table}"
                 )
                 continue
@@ -1438,7 +1611,13 @@ def get_upstream_tables(
                     table, default_schema_map=self.config.default_schema_map
                 )
             except Exception as e:
-                logger.info(f"Failed to generate upstream reference for {table}: {e}")
+                self.report.num_upstream_table_failed_generate_reference += 1
+                self.report.warning(
+                    title="Potentially Missing Lineage Issue",
+                    message="Failed to generate upstream reference",
+                    exc=e,
+                    context=f"table={table}",
+                )
                 continue
 
             table_urn = ref.make_dataset_urn(
@@ -1604,15 +1783,7 @@ def get_upstream_fields_from_custom_sql(
             func_overridden_info=None,  # Here we don't want to override any information from configuration
         )
 
-        if parsed_result is None:
-            logger.info(
-                f"Failed to extract column level lineage from datasource {datasource_urn}"
-            )
-            return []
-        if parsed_result.debug_info.error:
-            logger.info(
-                f"Failed to extract column level lineage from datasource {datasource_urn}: {parsed_result.debug_info.error}"
-            )
+        if parsed_result is None or parsed_result.debug_info.error:
             return []
 
         cll: List[ColumnLineageInfo] = (
@@ -1886,10 +2057,12 @@ def _query_published_datasource_for_project_luid(self, ds_luid: str) -> None:
                 self.datasource_project_map[ds_result.id] = ds_result.project_id
         except Exception as e:
             self.report.num_get_datasource_query_failures += 1
-            logger.warning(
-                f"Failed to get datasource project_luid for {ds_luid} due to error {e}"
+            self.report.warning(
+                title="Unexpected Query Error",
+                message="Failed to get datasource details",
+                exc=e,
+                context=f"ds_luid={ds_luid}",
             )
-            logger.debug("Error stack trace", exc_info=True)
 
     def _get_workbook_project_luid(self, wb: dict) -> Optional[str]:
         if wb.get(c.LUID) and self.workbook_project_map.get(wb[c.LUID]):
@@ -1974,6 +2147,8 @@ def _create_lineage_to_upstream_tables(
                 aspect_name=c.UPSTREAM_LINEAGE,
                 aspect=upstream_lineage,
             )
+            self.report.num_tables_with_upstream_lineage += 1
+            self.report.num_upstream_table_lineage += len(upstream_tables)
 
     @staticmethod
     def _clean_tableau_query_parameters(query: str) -> str:
@@ -2073,7 +2248,7 @@ def parse_custom_sql(
             f"Overridden info upstream_db={upstream_db}, platform_instance={platform_instance}, platform={platform}"
         )
 
-        return create_lineage_sql_parsed_result(
+        parsed_result = create_lineage_sql_parsed_result(
             query=query,
             default_db=upstream_db,
             platform=platform,
@@ -2083,10 +2258,24 @@ def parse_custom_sql(
             schema_aware=not self.config.sql_parsing_disable_schema_awareness,
         )
 
+        assert parsed_result is not None
+
+        if parsed_result.debug_info.table_error:
+            logger.warning(
+                f"Failed to extract table lineage from datasource {datasource_urn}: {parsed_result.debug_info.table_error}"
+            )
+            self.report.num_upstream_table_lineage_failed_parse_sql += 1
+        elif parsed_result.debug_info.column_error:
+            logger.warning(
+                f"Failed to extract column level lineage from datasource {datasource_urn}: {parsed_result.debug_info.column_error}"
+            )
+            self.report.num_upstream_fine_grained_lineage_failed_parse_sql += 1
+
+        return parsed_result
+
     def _enrich_database_tables_with_parsed_schemas(
         self, parsing_result: SqlParsingResult
     ) -> None:
-
         in_tables_schemas: Dict[
             str, Set[str]
         ] = transform_parsing_result_to_in_tables_schemas(parsing_result)
@@ -2118,9 +2307,6 @@ def _create_lineage_from_unsupported_csql(
         )
 
         if parsed_result is None:
-            logger.info(
-                f"Failed to extract table level lineage for datasource {csql_urn}"
-            )
             return
 
         self._enrich_database_tables_with_parsed_schemas(parsed_result)
@@ -2131,7 +2317,7 @@ def _create_lineage_from_unsupported_csql(
 
         fine_grained_lineages: List[FineGrainedLineage] = []
         if self.config.extract_column_level_lineage:
-            logger.info("Extracting CLL from custom sql")
+            logger.debug("Extracting CLL from custom sql")
             fine_grained_lineages = make_fine_grained_lineage_class(
                 parsed_result, csql_urn, out_columns
             )
@@ -2140,12 +2326,14 @@ def _create_lineage_from_unsupported_csql(
             upstreams=upstream_tables,
             fineGrainedLineages=fine_grained_lineages,
         )
-
         yield self.get_metadata_change_proposal(
             csql_urn,
             aspect_name=c.UPSTREAM_LINEAGE,
             aspect=upstream_lineage,
         )
+        self.report.num_tables_with_upstream_lineage += 1
+        self.report.num_upstream_table_lineage += len(upstream_tables)
+        self.report.num_upstream_fine_grained_lineage += len(fine_grained_lineages)
 
     def _get_schema_metadata_for_datasource(
         self, datasource_fields: List[dict]
@@ -2222,12 +2410,11 @@ def emit_datasource(
         )
 
         # Tags
-        if datasource_info:
+        if datasource_info and self.config.ingest_tags:
             tags = self.get_tags(datasource_info)
-            if tags:
-                dataset_snapshot.aspects.append(
-                    builder.make_global_tag_aspect_with_tag_list(tags)
-                )
+            dataset_snapshot.aspects.append(
+                builder.make_global_tag_aspect_with_tag_list(tags)
+            )
 
         # Browse path
         if browse_path and is_embedded_ds and workbook and workbook.get(c.NAME):
@@ -2296,6 +2483,11 @@ def emit_datasource(
                     aspect_name=c.UPSTREAM_LINEAGE,
                     aspect=upstream_lineage,
                 )
+                self.report.num_tables_with_upstream_lineage += 1
+                self.report.num_upstream_table_lineage += len(upstream_tables)
+                self.report.num_upstream_fine_grained_lineage += len(
+                    fine_grained_lineages
+                )
 
         # Datasource Fields
         schema_metadata = self._get_schema_metadata_for_datasource(
@@ -2613,7 +2805,13 @@ def emit_sheets(self) -> Iterable[MetadataWorkUnit]:
             c.SHEETS_CONNECTION,
             sheets_filter,
         ):
-            yield from self.emit_sheets_as_charts(sheet, sheet.get(c.WORKBOOK))
+            if self.config.ingest_hidden_assets or not self._is_hidden_view(sheet):
+                yield from self.emit_sheets_as_charts(sheet, sheet.get(c.WORKBOOK))
+            else:
+                self.report.num_hidden_assets_skipped += 1
+                logger.debug(
+                    f"Skip view {sheet.get(c.ID)} because it's hidden (luid is blank)."
+                )
 
     def emit_sheets_as_charts(
         self, sheet: dict, workbook: Optional[Dict]
@@ -2704,11 +2902,17 @@ def emit_sheets_as_charts(
             chart_snapshot.aspects.append(owner)
 
         #  Tags
-        tags = self.get_tags(sheet)
-        if tags:
+        if self.config.ingest_tags:
+            tags = self.get_tags(sheet)
+            if len(self.config.tags_for_hidden_assets) > 0 and self._is_hidden_view(
+                sheet
+            ):
+                tags.extend(self.config.tags_for_hidden_assets)
+
             chart_snapshot.aspects.append(
                 builder.make_global_tag_aspect_with_tag_list(tags)
             )
+
         yield self.get_metadata_change_event(chart_snapshot)
         if sheet_external_url is not None and self.config.ingest_embed_url is True:
             yield self.new_work_unit(
@@ -2790,7 +2994,7 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni
             else None
         )
 
-        tags = self.get_tags(workbook)
+        tags = self.get_tags(workbook) if self.config.ingest_tags else None
 
         parent_key = None
         project_luid: Optional[str] = self._get_workbook_project_luid(workbook)
@@ -2803,6 +3007,18 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni
                 f"Could not load project hierarchy for workbook {workbook_name}({workbook_id}). Please check permissions."
             )
 
+        custom_props = None
+        if (
+            self.config.permission_ingestion
+            and self.config.permission_ingestion.enable_workbooks
+        ):
+            logger.debug(f"Ingest access roles of workbook-id='{workbook.get(c.LUID)}'")
+            workbook_instance = self.server.workbooks.get_by_id(workbook.get(c.LUID))
+            self.server.workbooks.populate_permissions(workbook_instance)
+            custom_props = self._create_workbook_properties(
+                workbook_instance.permissions
+            )
+
         yield from gen_containers(
             container_key=workbook_container_key,
             name=workbook.get(c.NAME) or "",
@@ -2811,6 +3027,7 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni
             sub_types=[BIContainerSubTypes.TABLEAU_WORKBOOK],
             owner_urn=owner_urn,
             external_url=workbook_external_url,
+            extra_properties=custom_props,
             tags=tags,
         )
 
@@ -2908,17 +3125,23 @@ def emit_dashboards(self) -> Iterable[MetadataWorkUnit]:
             c.DASHBOARDS_CONNECTION,
             dashboards_filter,
         ):
-            yield from self.emit_dashboard(dashboard, dashboard.get(c.WORKBOOK))
+            if self.config.ingest_hidden_assets or not self._is_hidden_view(dashboard):
+                yield from self.emit_dashboard(dashboard, dashboard.get(c.WORKBOOK))
+            else:
+                self.report.num_hidden_assets_skipped += 1
+                logger.debug(
+                    f"Skip dashboard {dashboard.get(c.ID)} because it's hidden (luid is blank)."
+                )
 
-    def get_tags(self, obj: dict) -> Optional[List[str]]:
+    def get_tags(self, obj: dict) -> List[str]:
         tag_list = obj.get(c.TAGS, [])
-        if tag_list and self.config.ingest_tags:
+        if tag_list:
             tag_list_str = [
                 t[c.NAME] for t in tag_list if t is not None and t.get(c.NAME)
             ]
 
             return tag_list_str
-        return None
+        return []
 
     def emit_dashboard(
         self, dashboard: dict, workbook: Optional[Dict]
@@ -2969,8 +3192,13 @@ def emit_dashboard(
         )
         dashboard_snapshot.aspects.append(dashboard_info_class)
 
-        tags = self.get_tags(dashboard)
-        if tags:
+        if self.config.ingest_tags:
+            tags = self.get_tags(dashboard)
+            if len(self.config.tags_for_hidden_assets) > 0 and self._is_hidden_view(
+                dashboard
+            ):
+                tags.extend(self.config.tags_for_hidden_assets)
+
             dashboard_snapshot.aspects.append(
                 builder.make_global_tag_aspect_with_tag_list(tags)
             )
@@ -3138,10 +3366,10 @@ def emit_project_in_topological_order(
             else:
                 # This is a root Tableau project since the parent_project_id is None.
                 # For a root project, either the site is the parent, or the platform is the default parent.
-                if self.config.add_site_container and self.site and self.site.id:
+                if self.config.add_site_container:
                     # The site containers have already been generated by emit_site_container, so we
                     # don't need to emit them again here.
-                    parent_project_key = self.gen_site_key(self.site.id)
+                    parent_project_key = self.gen_site_key(self.site_id)
 
             yield from gen_containers(
                 container_key=project_key,
@@ -3158,21 +3386,63 @@ def emit_project_in_topological_order(
             yield from emit_project_in_topological_order(project)
 
     def emit_site_container(self):
-        if not self.site or not self.site.id:
+        if not self.site:
             logger.warning("Can not ingest site container. No site information found.")
             return
 
         yield from gen_containers(
-            container_key=self.gen_site_key(self.site.id),
+            container_key=self.gen_site_key(self.site_id),
             name=self.site.name or "Default",
             sub_types=[c.SITE],
         )
 
+    def _fetch_groups(self):
+        for group in TSC.Pager(self.server.groups):
+            self.group_map[group.id] = group
+
+    def _get_allowed_capabilities(self, capabilities: Dict[str, str]) -> List[str]:
+        if not self.config.permission_ingestion:
+            return []
+
+        allowed_capabilities = [
+            key for key, value in capabilities.items() if value == "Allow"
+        ]
+        return allowed_capabilities
+
+    def _create_workbook_properties(
+        self, permissions: List[PermissionsRule]
+    ) -> Optional[Dict[str, str]]:
+        if not self.config.permission_ingestion:
+            return None
+
+        groups = []
+        for rule in permissions:
+            if rule.grantee.tag_name == "group":
+                group = self.group_map.get(rule.grantee.id)
+                if not group or not group.name:
+                    logger.debug(f"Group {rule.grantee.id} not found in group map.")
+                    continue
+                if not self.config.permission_ingestion.group_name_pattern.allowed(
+                    group.name
+                ):
+                    logger.info(
+                        f"Skip permission '{group.name}' as it's excluded in group_name_pattern."
+                    )
+                    continue
+
+                capabilities = self._get_allowed_capabilities(rule.capabilities)
+                groups.append({"group": group.name, "capabilities": capabilities})
+
+        return {"permissions": json.dumps(groups)} if len(groups) > 0 else None
+
     def ingest_tableau_site(self):
         # Initialise the dictionary to later look-up for chart and dashboard stat
         if self.config.extract_usage_stats:
             self._populate_usage_stat_registry()
 
+        if self.config.permission_ingestion:
+            self._fetch_groups()
+
         # Populate the map of database names and database hostnames to be used later to map
         # databases to platform instances.
         if self.config.database_hostname_to_platform_instance_map:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py
index 8d6746b6433a4..61b56c4bee5bd 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py
@@ -643,8 +643,11 @@ def create(
         cls, d: dict, default_schema_map: Optional[Dict[str, str]] = None
     ) -> "TableauUpstreamReference":
         # Values directly from `table` object from Tableau
-        database = t_database = d.get(c.DATABASE, {}).get(c.NAME)
-        database_id = d.get(c.DATABASE, {}).get(c.ID)
+        database_dict = (
+            d.get(c.DATABASE) or {}
+        )  # this sometimes is None, so we need the `or {}`
+        database = t_database = database_dict.get(c.NAME)
+        database_id = database_dict.get(c.ID)
         schema = t_schema = d.get(c.SCHEMA)
         table = t_table = d.get(c.NAME) or ""
         t_full_name = d.get(c.FULL_NAME)
@@ -972,16 +975,22 @@ def get_filter_pages(query_filter: dict, page_size: int) -> List[dict]:
     # a few ten thousand, then tableau server responds with empty response
     # causing below error:
     # tableauserverclient.server.endpoint.exceptions.NonXMLResponseError: b''
+
+    # in practice, we only do pagination if len(query_filter.keys()) == 1
+    if len(query_filter.keys()) != 1:
+        return filter_pages
+
+    current_key = (list(query_filter.keys()))[0]
+
     if (
-        len(query_filter.keys()) == 1
-        and query_filter.get(c.ID_WITH_IN)
-        and isinstance(query_filter[c.ID_WITH_IN], list)
-        and len(query_filter[c.ID_WITH_IN]) > 100 * page_size
+        current_key in [c.ID_WITH_IN, c.PROJECT_NAME_WITH_IN]
+        and query_filter.get(current_key)
+        and isinstance(query_filter[current_key], list)
     ):
-        ids = query_filter[c.ID_WITH_IN]
+        ids = query_filter[current_key]
         filter_pages = [
             {
-                c.ID_WITH_IN: ids[
+                current_key: ids[
                     start : (
                         start + page_size if start + page_size < len(ids) else len(ids)
                     )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_constant.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_constant.py
index d1dd0d9281999..d69312f803021 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_constant.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_constant.py
@@ -81,3 +81,7 @@
 PROJECT = "Project"
 SITE = "Site"
 IS_UNSUPPORTED_CUSTOM_SQL = "isUnsupportedCustomSql"
+SITE_PERMISSION = "sitePermission"
+ROLE_SITE_ADMIN_EXPLORER = "SiteAdministratorExplorer"
+ROLE_SITE_ADMIN_CREATOR = "SiteAdministratorCreator"
+ROLE_SERVER_ADMIN = "ServerAdministrator"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_server_wrapper.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_server_wrapper.py
new file mode 100644
index 0000000000000..482140a227511
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_server_wrapper.py
@@ -0,0 +1,37 @@
+from dataclasses import dataclass
+
+from tableauserverclient import Server, UserItem
+
+from datahub.ingestion.source.tableau import tableau_constant as c
+
+
+@dataclass
+class UserInfo:
+    user_name: str
+    site_role: str
+    site_id: str
+
+    def has_site_administrator_explorer_privileges(self):
+        return self.site_role in [
+            c.ROLE_SITE_ADMIN_EXPLORER,
+            c.ROLE_SITE_ADMIN_CREATOR,
+            c.ROLE_SERVER_ADMIN,
+        ]
+
+    @staticmethod
+    def from_server(server: Server) -> "UserInfo":
+        assert server.user_id, "make the connection with tableau"
+
+        user: UserItem = server.users.get_by_id(server.user_id)
+
+        assert user.site_role, "site_role is not available"  # to silent the lint
+
+        assert user.name, "user name is not available"  # to silent the lint
+
+        assert server.site_id, "site identifier is not available"  # to silent the lint
+
+        return UserInfo(
+            user_name=user.name,
+            site_role=user.site_role,
+            site_id=server.site_id,
+        )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_validation.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_validation.py
new file mode 100644
index 0000000000000..4ec0e5ef01d3c
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_validation.py
@@ -0,0 +1,48 @@
+import logging
+from typing import Dict, Union
+
+from datahub.ingestion.api.source import CapabilityReport, SourceCapability
+from datahub.ingestion.source.tableau import tableau_constant as c
+from datahub.ingestion.source.tableau.tableau_server_wrapper import UserInfo
+
+logger = logging.getLogger(__name__)
+
+
+def check_user_role(
+    logged_in_user: UserInfo,
+) -> Dict[Union[SourceCapability, str], CapabilityReport]:
+    capability_dict: Dict[Union[SourceCapability, str], CapabilityReport] = {
+        c.SITE_PERMISSION: CapabilityReport(
+            capable=True,
+        )
+    }
+
+    failure_reason: str = (
+        "The user does not have the `Site Administrator Explorer` role."
+    )
+
+    mitigation_message_prefix: str = (
+        "Assign `Site Administrator Explorer` role to the user"
+    )
+    mitigation_message_suffix: str = "Refer to the setup guide: https://datahubproject.io/docs/quick-ingestion-guides/tableau/setup"
+
+    try:
+        # TODO: Add check for `Enable Derived Permissions`
+        if not logged_in_user.has_site_administrator_explorer_privileges():
+            capability_dict[c.SITE_PERMISSION] = CapabilityReport(
+                capable=False,
+                failure_reason=f"{failure_reason} Their current role is {logged_in_user.site_role}.",
+                mitigation_message=f"{mitigation_message_prefix} `{logged_in_user.user_name}`. {mitigation_message_suffix}",
+            )
+
+        return capability_dict
+
+    except Exception as e:
+        logger.warning(msg=e, exc_info=e)
+        capability_dict[c.SITE_PERMISSION] = CapabilityReport(
+            capable=False,
+            failure_reason="Failed to verify user role.",
+            mitigation_message=f"{mitigation_message_prefix}. {mitigation_message_suffix}",  # user is unknown
+        )
+
+        return capability_dict
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
index bd987c2da7c76..11827bace4b5a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
@@ -462,8 +462,6 @@ def _create_table(
                 datetime.fromtimestamp(obj.updated_at / 1000, tz=timezone.utc)
                 if obj.updated_at
                 else None
-                if obj.updated_at
-                else None
             ),
             updated_by=obj.updated_by,
             table_id=obj.table_id,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
index f84f6c1b0c08d..9c5752c518df1 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
@@ -33,6 +33,7 @@
 
 logger = logging.getLogger(__name__)
 
+# TODO: (maybe) Replace with standardized types in sql_types.py
 DATA_TYPE_REGISTRY: dict = {
     ColumnTypeName.BOOLEAN: BooleanTypeClass,
     ColumnTypeName.BYTE: BytesTypeClass,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
index a00a52ae54207..f16769341853a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
@@ -2,7 +2,7 @@
 from typing import Optional, Tuple
 
 from datahub.ingestion.api.report import EntityFilterReport, Report
-from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport
+from datahub.ingestion.source.sql.sql_report import SQLSourceReport
 from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.utilities.lossy_collections import LossyDict, LossyList
 from datahub.utilities.perf_timer import PerfTimer
@@ -12,12 +12,14 @@
 class UnityCatalogUsagePerfReport(Report):
     get_queries_timer: PerfTimer = field(default_factory=PerfTimer)
     sql_parsing_timer: PerfTimer = field(default_factory=PerfTimer)
+    spark_sql_parsing_timer: PerfTimer = field(default_factory=PerfTimer)
     aggregator_add_event_timer: PerfTimer = field(default_factory=PerfTimer)
     gen_operation_timer: PerfTimer = field(default_factory=PerfTimer)
+    query_fingerprinting_timer: PerfTimer = field(default_factory=PerfTimer)
 
 
 @dataclass
-class UnityCatalogReport(IngestionStageReport, ProfilingSqlReport):
+class UnityCatalogReport(IngestionStageReport, SQLSourceReport):
     metastores: EntityFilterReport = EntityFilterReport.field(type="metastore")
     catalogs: EntityFilterReport = EntityFilterReport.field(type="catalog")
     schemas: EntityFilterReport = EntityFilterReport.field(type="schema")
@@ -32,6 +34,7 @@ class UnityCatalogReport(IngestionStageReport, ProfilingSqlReport):
     num_external_upstreams_unsupported: int = 0
 
     num_queries: int = 0
+    num_unique_queries: int = 0
     num_queries_dropped_parse_failure: int = 0
     num_queries_missing_table: int = 0  # Can be due to pattern filter
     num_queries_duplicate_table: int = 0
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
index 9a6cde78cf10d..7bfa7fdb28aaf 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
@@ -26,6 +26,9 @@
     gen_containers,
 )
 from datahub.emitter.sql_parsing_builder import SqlParsingBuilder
+from datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size import (
+    EnsureAspectSizeProcessor,
+)
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
@@ -260,6 +263,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
             ).workunit_processor,
+            EnsureAspectSizeProcessor(self.get_report()).ensure_aspect_size,
         ]
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
@@ -556,6 +560,8 @@ def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUn
                 )
 
         if table_props:
+            # TODO: use auto_incremental_properties workunit processor instead
+            # Consider enabling incremental_properties by default
             patch_builder = create_dataset_props_patch_builder(dataset_urn, table_props)
             for patch_mcp in patch_builder.build():
                 yield MetadataWorkUnit(
@@ -974,6 +980,8 @@ def _run_sql_parser(
             )
         else:
             self.report.num_view_definitions_parsed += 1
+            if raw_lineage.out_tables != [view_urn]:
+                self.report.num_view_definitions_view_urn_mismatch += 1
         return view_definition_lineage_helper(raw_lineage, view_urn)
 
     def get_view_lineage(self) -> Iterable[MetadataWorkUnit]:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py b/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py
index 08482c9d2fa3b..718818d9b347b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py
@@ -7,7 +7,6 @@
 
 import pyspark
 from databricks.sdk.service.sql import QueryStatementType
-from sqllineage.runner import LineageRunner
 
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.source_helpers import auto_empty_dataset_usage_statistics
@@ -22,6 +21,9 @@
 from datahub.ingestion.source.unity.report import UnityCatalogReport
 from datahub.ingestion.source.usage.usage_common import UsageAggregator
 from datahub.metadata.schema_classes import OperationClass
+from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
+from datahub.sql_parsing.sqlglot_utils import get_query_fingerprint
+from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 logger = logging.getLogger(__name__)
 
@@ -47,6 +49,7 @@ class UnityCatalogUsageExtractor:
     proxy: UnityCatalogApiProxy
     table_urn_builder: Callable[[TableReference], str]
     user_urn_builder: Callable[[str], str]
+    platform: str = "databricks"
 
     def __post_init__(self):
         self.usage_aggregator = UsageAggregator[TableReference](self.config)
@@ -76,6 +79,7 @@ def _get_workunits_internal(
         self, table_refs: Set[TableReference]
     ) -> Iterable[MetadataWorkUnit]:
         table_map = defaultdict(list)
+        query_hashes = set()
         for ref in table_refs:
             table_map[ref.table].append(ref)
             table_map[f"{ref.schema}.{ref.table}"].append(ref)
@@ -85,6 +89,13 @@ def _get_workunits_internal(
             for query in self._get_queries():
                 self.report.num_queries += 1
                 with current_timer.pause():
+                    with self.report.usage_perf_report.query_fingerprinting_timer:
+                        query_hashes.add(
+                            get_query_fingerprint(
+                                query.query_text, "databricks", fast=True
+                            )
+                        )
+                        self.report.num_unique_queries = len(query_hashes)
                     table_info = self._parse_query(query, table_map)
                     if table_info is not None:
                         if self.config.include_operational_stats:
@@ -164,9 +175,10 @@ def _parse_query(
         self, query: Query, table_map: TableMap
     ) -> Optional[QueryTableInfo]:
         with self.report.usage_perf_report.sql_parsing_timer:
-            table_info = self._parse_query_via_lineage_runner(query.query_text)
+            table_info = self._parse_query_via_sqlglot(query.query_text)
             if table_info is None and query.statement_type == QueryStatementType.SELECT:
-                table_info = self._parse_query_via_spark_sql_plan(query.query_text)
+                with self.report.usage_perf_report.spark_sql_parsing_timer:
+                    table_info = self._parse_query_via_spark_sql_plan(query.query_text)
 
             if table_info is None:
                 self.report.num_queries_dropped_parse_failure += 1
@@ -181,26 +193,33 @@ def _parse_query(
                     ),
                 )
 
-    def _parse_query_via_lineage_runner(self, query: str) -> Optional[StringTableInfo]:
+    def _parse_query_via_sqlglot(self, query: str) -> Optional[StringTableInfo]:
         try:
-            runner = LineageRunner(query)
+            sql_parser_in_tables = create_lineage_sql_parsed_result(
+                query=query,
+                default_db=None,
+                platform=self.platform,
+                env=self.config.env,
+                platform_instance=None,
+            )
+
             return GenericTableInfo(
                 source_tables=[
-                    self._parse_sqllineage_table(table)
-                    for table in runner.source_tables
+                    self._parse_sqlglot_table(table)
+                    for table in sql_parser_in_tables.in_tables
                 ],
                 target_tables=[
-                    self._parse_sqllineage_table(table)
-                    for table in runner.target_tables
+                    self._parse_sqlglot_table(table)
+                    for table in sql_parser_in_tables.out_tables
                 ],
             )
         except Exception as e:
-            logger.info(f"Could not parse query via lineage runner, {query}: {e!r}")
+            logger.info(f"Could not parse query via sqlglot, {query}: {e!r}")
             return None
 
     @staticmethod
-    def _parse_sqllineage_table(sqllineage_table: object) -> str:
-        full_table_name = str(sqllineage_table)
+    def _parse_sqlglot_table(table_urn: str) -> str:
+        full_table_name = DatasetUrn.from_string(table_urn).name
         default_schema = "<default>."
         if full_table_name.startswith(default_schema):
             return full_table_name[len(default_schema) :]
diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py b/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py
index 4308b405e46e3..92407eaae6e90 100644
--- a/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py
@@ -14,6 +14,7 @@
 USAGE_EXTRACTION_INGESTION = "Usage Extraction Ingestion"
 USAGE_EXTRACTION_OPERATIONAL_STATS = "Usage Extraction Operational Stats"
 USAGE_EXTRACTION_USAGE_AGGREGATION = "Usage Extraction Usage Aggregation"
+EXTERNAL_TABLE_DDL_LINEAGE = "External table DDL Lineage"
 QUERIES_EXTRACTION = "Queries Extraction"
 PROFILING = "Profiling"
 
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_dataproduct.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_dataproduct.py
index ce224bde003fd..bb1c297513de1 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_dataproduct.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_dataproduct.py
@@ -105,7 +105,6 @@ class SimpleAddDatasetDataProduct(AddDatasetDataProduct):
     """Transformer that adds a specified dataproduct entity for provided dataset as its asset."""
 
     def __init__(self, config: SimpleDatasetDataProductConfig, ctx: PipelineContext):
-
         generic_config = AddDatasetDataProductConfig(
             get_data_product_to_add=lambda dataset_urn: config.dataset_to_data_product_urns.get(
                 dataset_urn
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_ownership.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_ownership.py
index 54be2e5fac1e3..b107a62c905b4 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_ownership.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_ownership.py
@@ -13,9 +13,7 @@
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.graph.client import DataHubGraph
-from datahub.ingestion.transformer.dataset_transformer import (
-    DatasetOwnershipTransformer,
-)
+from datahub.ingestion.transformer.dataset_transformer import OwnershipTransformer
 from datahub.metadata.schema_classes import (
     BrowsePathsV2Class,
     MetadataChangeProposalClass,
@@ -37,7 +35,7 @@ class AddDatasetOwnershipConfig(TransformerSemanticsConfigModel):
     is_container: bool = False
 
 
-class AddDatasetOwnership(DatasetOwnershipTransformer):
+class AddDatasetOwnership(OwnershipTransformer):
     """Transformer that adds owners to datasets according to a callback function."""
 
     ctx: PipelineContext
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py
index ef6ef43fa2d7f..355ca7a373653 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py
@@ -67,7 +67,6 @@ def transform_aspect(
     def handle_end_of_stream(
         self,
     ) -> List[Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]]:
-
         mcps: List[
             Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]
         ] = []
@@ -75,7 +74,7 @@ def handle_end_of_stream(
         logger.debug("Generating tags")
 
         for tag_association in self.processed_tags.values():
-            tag_urn = TagUrn.create_from_string(tag_association.tag)
+            tag_urn = TagUrn.from_string(tag_association.tag)
             mcps.append(
                 MetadataChangeProposalWrapper(
                     entityUrn=tag_urn.urn(),
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_domain.py b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_domain.py
index 6a83824815265..6b78b71eaa78e 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_domain.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_domain.py
@@ -1,6 +1,8 @@
 import logging
+from enum import auto
 from typing import Callable, Dict, List, Optional, Sequence, Union, cast
 
+from datahub.configuration._config_enum import ConfigEnum
 from datahub.configuration.common import (
     ConfigurationError,
     KeyValuePattern,
@@ -23,6 +25,13 @@
 logger = logging.getLogger(__name__)
 
 
+class TransformerOnConflict(ConfigEnum):
+    """Describes the behavior of the transformer when writing an aspect that already exists."""
+
+    DO_UPDATE = auto()  # On conflict, apply the new aspect
+    DO_NOTHING = auto()  # On conflict, do not apply the new aspect
+
+
 class AddDatasetDomainSemanticsConfig(TransformerSemanticsConfigModel):
     get_domains_to_add: Union[
         Callable[[str], DomainsClass],
@@ -32,10 +41,12 @@ class AddDatasetDomainSemanticsConfig(TransformerSemanticsConfigModel):
     _resolve_domain_fn = pydantic_resolve_key("get_domains_to_add")
 
     is_container: bool = False
+    on_conflict: TransformerOnConflict = TransformerOnConflict.DO_UPDATE
 
 
 class SimpleDatasetDomainSemanticsConfig(TransformerSemanticsConfigModel):
     domains: List[str]
+    on_conflict: TransformerOnConflict = TransformerOnConflict.DO_UPDATE
 
 
 class PatternDatasetDomainSemanticsConfig(TransformerSemanticsConfigModel):
@@ -80,12 +91,13 @@ def get_domain_class(
 
     @staticmethod
     def _merge_with_server_domains(
-        graph: DataHubGraph, urn: str, mce_domain: Optional[DomainsClass]
+        graph: Optional[DataHubGraph], urn: str, mce_domain: Optional[DomainsClass]
     ) -> Optional[DomainsClass]:
         if not mce_domain or not mce_domain.domains:
             # nothing to add, no need to consult server
             return None
 
+        assert graph
         server_domain = graph.get_domain(entity_urn=urn)
         if server_domain:
             # compute patch
@@ -155,7 +167,7 @@ def transform_aspect(
         self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect]
     ) -> Optional[Aspect]:
         in_domain_aspect: DomainsClass = cast(DomainsClass, aspect)
-        domain_aspect = DomainsClass(domains=[])
+        domain_aspect: DomainsClass = DomainsClass(domains=[])
         # Check if we have received existing aspect
         if in_domain_aspect is not None and self.config.replace_existing is False:
             domain_aspect.domains.extend(in_domain_aspect.domains)
@@ -164,16 +176,18 @@ def transform_aspect(
 
         domain_aspect.domains.extend(domain_to_add.domains)
 
-        if self.config.semantics == TransformerSemantics.PATCH:
-            assert self.ctx.graph
-            patch_domain_aspect: Optional[
-                DomainsClass
-            ] = AddDatasetDomain._merge_with_server_domains(
-                self.ctx.graph, entity_urn, domain_aspect
-            )
-            return cast(Optional[Aspect], patch_domain_aspect)
-
-        return cast(Optional[Aspect], domain_aspect)
+        final_aspect: Optional[DomainsClass] = domain_aspect
+        if domain_aspect.domains:
+            if self.config.on_conflict == TransformerOnConflict.DO_NOTHING:
+                assert self.ctx.graph
+                server_domain = self.ctx.graph.get_domain(entity_urn)
+                if server_domain and server_domain.domains:
+                    return None
+            if self.config.semantics == TransformerSemantics.PATCH:
+                final_aspect = AddDatasetDomain._merge_with_server_domains(
+                    self.ctx.graph, entity_urn, domain_aspect
+                )
+        return cast(Optional[Aspect], final_aspect)
 
 
 class SimpleAddDatasetDomain(AddDatasetDomain):
@@ -186,8 +200,7 @@ def __init__(
         domains = AddDatasetDomain.get_domain_class(ctx.graph, config.domains)
         generic_config = AddDatasetDomainSemanticsConfig(
             get_domains_to_add=lambda _: domains,
-            semantics=config.semantics,
-            replace_existing=config.replace_existing,
+            **config.dict(exclude={"domains"}),
         )
         super().__init__(generic_config, ctx)
 
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py
index 42dd54f4a584a..00b3a9ba59f92 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py
@@ -27,6 +27,22 @@ def entity_types(self) -> List[str]:
         return ["dataset"]
 
 
+class OwnershipTransformer(
+    DatasetTransformer, SingleAspectTransformer, metaclass=ABCMeta
+):
+    def aspect_name(self) -> str:
+        return "ownership"
+
+    def entity_types(self) -> List[str]:
+        return [
+            "dataset",
+            "dataJob",
+            "dataFlow",
+            "chart",
+            "dashboard",
+        ]
+
+
 class TagTransformer(BaseTransformer, SingleAspectTransformer, metaclass=ABCMeta):
     """Transformer that does transform sequentially on each tag."""
 
@@ -47,11 +63,6 @@ def entity_types(self) -> List[str]:
         return ["container"]
 
 
-class DatasetOwnershipTransformer(DatasetTransformer, metaclass=ABCMeta):
-    def aspect_name(self) -> str:
-        return "ownership"
-
-
 class DatasetDomainTransformer(DatasetTransformer, metaclass=ABCMeta):
     def aspect_name(self) -> str:
         return "domains"
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py
index 245a3aa3d9db1..212e018dd64fb 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py
@@ -105,7 +105,6 @@ def convert_tag_as_per_mapping(self, tag: str) -> str:
     def handle_end_of_stream(
         self,
     ) -> Sequence[Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]]:
-
         return self.owner_mcps
 
     def transform_aspect(
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py
index 3c0bc00633e7c..5bf70274dce89 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py
@@ -100,7 +100,7 @@ def transform(
                         )
                         if transformed_aspect:
                             # for end of stream records, we modify the workunit-id
-                            structured_urn = Urn.create_from_string(urn)
+                            structured_urn = Urn.from_string(urn)
                             simple_name = "-".join(structured_urn.get_entity_id())
                             record_metadata = envelope.metadata.copy()
                             record_metadata.update(
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/pattern_cleanup_ownership.py b/metadata-ingestion/src/datahub/ingestion/transformer/pattern_cleanup_ownership.py
index 8ef61ab9679e6..f17546d6f7299 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/pattern_cleanup_ownership.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/pattern_cleanup_ownership.py
@@ -4,9 +4,7 @@
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import ConfigModel
 from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.transformer.dataset_transformer import (
-    DatasetOwnershipTransformer,
-)
+from datahub.ingestion.transformer.dataset_transformer import OwnershipTransformer
 from datahub.metadata.schema_classes import (
     OwnerClass,
     OwnershipClass,
@@ -20,7 +18,7 @@ class PatternCleanUpOwnershipConfig(ConfigModel):
     pattern_for_cleanup: List[str]
 
 
-class PatternCleanUpOwnership(DatasetOwnershipTransformer):
+class PatternCleanUpOwnership(OwnershipTransformer):
     """Transformer that clean the ownership URN."""
 
     ctx: PipelineContext
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/remove_dataset_ownership.py b/metadata-ingestion/src/datahub/ingestion/transformer/remove_dataset_ownership.py
index f5d71a4340554..934e2a13d5631 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/remove_dataset_ownership.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/remove_dataset_ownership.py
@@ -3,9 +3,7 @@
 from datahub.configuration.common import ConfigModel
 from datahub.emitter.mce_builder import Aspect
 from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.transformer.dataset_transformer import (
-    DatasetOwnershipTransformer,
-)
+from datahub.ingestion.transformer.dataset_transformer import OwnershipTransformer
 from datahub.metadata.schema_classes import OwnershipClass
 
 
@@ -13,7 +11,7 @@ class ClearDatasetOwnershipConfig(ConfigModel):
     pass
 
 
-class SimpleRemoveDatasetOwnership(DatasetOwnershipTransformer):
+class SimpleRemoveDatasetOwnership(OwnershipTransformer):
     """Transformer that clears all owners on each dataset."""
 
     def __init__(self, config: ClearDatasetOwnershipConfig, ctx: PipelineContext):
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/replace_external_url.py b/metadata-ingestion/src/datahub/ingestion/transformer/replace_external_url.py
index 57af10d1040c8..f6847f234aefe 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/replace_external_url.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/replace_external_url.py
@@ -103,7 +103,6 @@ def create(
     def transform_aspect(
         self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect]
     ) -> Optional[Aspect]:
-
         in_container_properties_aspect: ContainerPropertiesClass = cast(
             ContainerPropertiesClass, aspect
         )
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/system_metadata_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/system_metadata_transformer.py
index 8a0636f5a11a9..3b5f26c127741 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/system_metadata_transformer.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/system_metadata_transformer.py
@@ -31,14 +31,14 @@ def auto_system_metadata(
 
 class SystemMetadataTransformer(Transformer):
     def __init__(self, ctx: PipelineContext):
-        self._inner_transfomer = AutoHelperTransformer(
+        self._inner_transformer = AutoHelperTransformer(
             functools.partial(auto_system_metadata, ctx)
         )
 
     def transform(
         self, record_envelopes: Iterable[RecordEnvelope]
     ) -> Iterable[RecordEnvelope]:
-        yield from self._inner_transfomer.transform(record_envelopes)
+        yield from self._inner_transformer.transform(record_envelopes)
 
     @classmethod
     def create(cls, config_dict: dict, ctx: PipelineContext) -> Transformer:
diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py b/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py
index 338f191c0829d..7e6125079f16e 100644
--- a/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py
+++ b/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py
@@ -84,7 +84,6 @@ def get_tags_from_schema_metadata(
     def transform_aspect(
         self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect]
     ) -> Optional[Aspect]:
-
         in_glossary_terms: Optional[GlossaryTermsClass] = cast(
             Optional[GlossaryTermsClass], aspect
         )
diff --git a/metadata-ingestion/src/datahub/integrations/assertion/common.py b/metadata-ingestion/src/datahub/integrations/assertion/common.py
index 9ffad5cf66640..f0b7510df14a3 100644
--- a/metadata-ingestion/src/datahub/integrations/assertion/common.py
+++ b/metadata-ingestion/src/datahub/integrations/assertion/common.py
@@ -42,7 +42,7 @@ def get_entity_name(assertion: BaseEntityAssertion) -> Tuple[str, str, str]:
         if qualified_name is not None:
             parts = qualified_name.split(".")
         else:
-            urn_id = Urn.create_from_string(assertion.entity).entity_ids[1]
+            urn_id = Urn.from_string(assertion.entity).entity_ids[1]
             parts = urn_id.split(".")
             if len(parts) > 3:
                 parts = parts[-3:]
diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py
index 5b079129e0a9c..facc7d107d1ba 100644
--- a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py
+++ b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py
@@ -72,7 +72,6 @@ def _(self, assertion: FixedIntervalFreshnessAssertion) -> str:
 
     @metric_sql.register
     def _(self, assertion: RowCountTotalVolumeAssertion) -> str:
-
         # Can not use information schema here due to error -
         # Data metric function body cannot refer to the non-deterministic function 'CURRENT_DATABASE_MAIN_METASTORE_ID'.
 
diff --git a/metadata-ingestion/src/datahub/lite/duckdb_lite.py b/metadata-ingestion/src/datahub/lite/duckdb_lite.py
index f40a2e9498e62..8931738352092 100644
--- a/metadata-ingestion/src/datahub/lite/duckdb_lite.py
+++ b/metadata-ingestion/src/datahub/lite/duckdb_lite.py
@@ -609,7 +609,7 @@ def get_typed_aspect(
             aspect_map, DataPlatformInstanceClass
         )  # type: ignore
 
-        needs_platform = Urn.create_from_string(entity_urn).get_type() in [
+        needs_platform = Urn.from_string(entity_urn).get_type() in [
             "dataset",
             "container",
             "chart",
@@ -617,7 +617,7 @@ def get_typed_aspect(
             "dataFlow",
             "dataJob",
         ]
-        entity_urn_parsed = Urn.create_from_string(entity_urn)
+        entity_urn_parsed = Urn.from_string(entity_urn)
         if entity_urn_parsed.get_type() in ["dataFlow", "dataJob"]:
             self.add_edge(
                 entity_urn,
@@ -630,15 +630,12 @@ def get_typed_aspect(
             # this is a top-level entity
             if not dpi:
                 logger.debug(f"No data platform instance for {entity_urn}")
-                maybe_parent_urn = Urn.create_from_string(entity_urn).get_entity_id()[0]
+                maybe_parent_urn = Urn.from_string(entity_urn).get_entity_id()[0]
                 needs_dpi = False
                 if maybe_parent_urn.startswith(Urn.URN_PREFIX):
                     parent_urn = maybe_parent_urn
-                    if (
-                        Urn.create_from_string(maybe_parent_urn).get_type()
-                        == "dataPlatform"
-                    ):
-                        data_platform_urn = DataPlatformUrn.create_from_string(
+                    if Urn.from_string(maybe_parent_urn).get_type() == "dataPlatform":
+                        data_platform_urn = DataPlatformUrn.from_string(
                             maybe_parent_urn
                         )
                         needs_dpi = True
@@ -660,7 +657,7 @@ def get_typed_aspect(
                         logger.error(f"Failed to generate edges entity {entity_urn}", e)
                     parent_urn = str(data_platform_instance_urn)
             else:
-                data_platform_urn = DataPlatformUrn.create_from_string(dpi.platform)
+                data_platform_urn = DataPlatformUrn.from_string(dpi.platform)
                 data_platform_instance = dpi.instance or "default"
                 data_platform_instance_urn = Urn(
                     entity_type="dataPlatformInstance",
@@ -673,9 +670,7 @@ def get_typed_aspect(
             parent_urn = "__root__"
 
         types = (
-            subtypes.typeNames
-            if subtypes
-            else [Urn.create_from_string(entity_urn).get_type()]
+            subtypes.typeNames if subtypes else [Urn.from_string(entity_urn).get_type()]
         )
         for t in types:
             type_urn = Urn(entity_type="systemNode", entity_id=[parent_urn, t])
@@ -686,7 +681,7 @@ def get_typed_aspect(
     def _create_edges_from_data_platform_instance(
         self, data_platform_instance_urn: Urn
     ) -> None:
-        data_platform_urn = DataPlatformUrn.create_from_string(
+        data_platform_urn = DataPlatformUrn.from_string(
             data_platform_instance_urn.get_entity_id()[0]
         )
         data_platform_instances_urn = Urn(
@@ -735,7 +730,7 @@ def post_update_hook(
         if isinstance(aspect, DatasetPropertiesClass):
             dp: DatasetPropertiesClass = aspect
             if dp.name:
-                specific_urn = DatasetUrn.create_from_string(entity_urn)
+                specific_urn = DatasetUrn.from_string(entity_urn)
                 if (
                     specific_urn.get_data_platform_urn().get_entity_id_as_string()
                     == "looker"
@@ -755,7 +750,7 @@ def post_update_hook(
             self.add_edge(entity_urn, "name", cp.name, remove_existing=True)
         elif isinstance(aspect, DataPlatformInstanceClass):
             dpi: DataPlatformInstanceClass = aspect
-            data_platform_urn = DataPlatformUrn.create_from_string(dpi.platform)
+            data_platform_urn = DataPlatformUrn.from_string(dpi.platform)
             data_platform_instance = dpi.instance or "default"
             data_platform_instance_urn = Urn(
                 entity_type="dataPlatformInstance",
@@ -763,7 +758,7 @@ def post_update_hook(
             )
             self._create_edges_from_data_platform_instance(data_platform_instance_urn)
         elif isinstance(aspect, ChartInfoClass):
-            urn = Urn.create_from_string(entity_urn)
+            urn = Urn.from_string(entity_urn)
             self.add_edge(
                 entity_urn,
                 "name",
@@ -771,7 +766,7 @@ def post_update_hook(
                 remove_existing=True,
             )
         elif isinstance(aspect, DashboardInfoClass):
-            urn = Urn.create_from_string(entity_urn)
+            urn = Urn.from_string(entity_urn)
             self.add_edge(
                 entity_urn,
                 "name",
diff --git a/metadata-ingestion/src/datahub/specific/chart.py b/metadata-ingestion/src/datahub/specific/chart.py
index cc68168b68db7..104a7c21a07e2 100644
--- a/metadata-ingestion/src/datahub/specific/chart.py
+++ b/metadata-ingestion/src/datahub/specific/chart.py
@@ -1,10 +1,8 @@
-import time
 from typing import Dict, List, Optional, Union
 
 from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
 from datahub.metadata.schema_classes import (
     AccessLevelClass,
-    AuditStampClass,
     ChangeAuditStampsClass,
     ChartInfoClass as ChartInfo,
     ChartTypeClass,
@@ -47,43 +45,6 @@ def __init__(
         )
         self.ownership_patch_helper = OwnershipPatchHelper(self)
 
-    def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass:
-        """
-        Creates an AuditStampClass instance with the current timestamp and other default values.
-
-        Args:
-            message: The message associated with the audit stamp (optional).
-
-        Returns:
-            An instance of AuditStampClass.
-        """
-        return AuditStampClass(
-            time=int(time.time() * 1000.0),
-            actor="urn:li:corpuser:datahub",
-            message=message,
-        )
-
-    def _ensure_urn_type(
-        self, entity_type: str, edges: List[Edge], context: str
-    ) -> None:
-        """
-        Ensures that the destination URNs in the given edges have the specified entity type.
-
-        Args:
-            entity_type: The entity type to check against.
-            edges: A list of Edge objects.
-            context: The context or description of the operation.
-
-        Raises:
-            ValueError: If any of the destination URNs is not of the specified entity type.
-        """
-        for e in edges:
-            urn = Urn.create_from_string(e.destinationUrn)
-            if not urn.get_type() == entity_type:
-                raise ValueError(
-                    f"{context}: {e.destinationUrn} is not of type {entity_type}"
-                )
-
     def add_owner(self, owner: Owner) -> "ChartPatchBuilder":
         """
         Adds an owner to the ChartPatchBuilder.
diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py
index 8228dbc011db2..da5abbfd1dc12 100644
--- a/metadata-ingestion/src/datahub/specific/dashboard.py
+++ b/metadata-ingestion/src/datahub/specific/dashboard.py
@@ -1,10 +1,8 @@
-import time
 from typing import Dict, List, Optional, Union
 
 from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
 from datahub.metadata.schema_classes import (
     AccessLevelClass,
-    AuditStampClass,
     ChangeAuditStampsClass,
     DashboardInfoClass as DashboardInfo,
     EdgeClass as Edge,
@@ -46,43 +44,6 @@ def __init__(
         )
         self.ownership_patch_helper = OwnershipPatchHelper(self)
 
-    def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass:
-        """
-        Creates an AuditStampClass instance with the current timestamp and other default values.
-
-        Args:
-            message: The message associated with the audit stamp (optional).
-
-        Returns:
-            An instance of AuditStampClass.
-        """
-        return AuditStampClass(
-            time=int(time.time() * 1000.0),
-            actor="urn:li:corpuser:datahub",
-            message=message,
-        )
-
-    def _ensure_urn_type(
-        self, entity_type: str, edges: List[Edge], context: str
-    ) -> None:
-        """
-        Ensures that the destination URNs in the given edges have the specified entity type.
-
-        Args:
-            entity_type: The entity type to check against.
-            edges: A list of Edge objects.
-            context: The context or description of the operation.
-
-        Raises:
-            ValueError: If any of the destination URNs is not of the specified entity type.
-        """
-        for e in edges:
-            urn = Urn.create_from_string(e.destinationUrn)
-            if not urn.get_type() == entity_type:
-                raise ValueError(
-                    f"{context}: {e.destinationUrn} is not of type {entity_type}"
-                )
-
     def add_owner(self, owner: Owner) -> "DashboardPatchBuilder":
         """
         Adds an owner to the DashboardPatchBuilder.
@@ -433,7 +394,6 @@ def set_description(self, description: str) -> "DashboardPatchBuilder":
     def add_custom_properties(
         self, custom_properties: Optional[Dict[str, str]] = None
     ) -> "DashboardPatchBuilder":
-
         if custom_properties:
             for key, value in custom_properties.items():
                 self.custom_properties_patch_helper.add_property(key, value)
diff --git a/metadata-ingestion/src/datahub/specific/datajob.py b/metadata-ingestion/src/datahub/specific/datajob.py
index 8da8edc8ef0f2..6ff4741b09c26 100644
--- a/metadata-ingestion/src/datahub/specific/datajob.py
+++ b/metadata-ingestion/src/datahub/specific/datajob.py
@@ -1,9 +1,7 @@
-import time
 from typing import Dict, List, Optional, Union
 
 from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
 from datahub.metadata.schema_classes import (
-    AuditStampClass,
     DataJobInfoClass as DataJobInfo,
     DataJobInputOutputClass as DataJobInputOutput,
     EdgeClass as Edge,
@@ -16,10 +14,9 @@
     SystemMetadataClass,
     TagAssociationClass as Tag,
 )
+from datahub.metadata.urns import SchemaFieldUrn, TagUrn, Urn
 from datahub.specific.custom_properties import CustomPropertiesPatchHelper
 from datahub.specific.ownership import OwnershipPatchHelper
-from datahub.utilities.urns.tag_urn import TagUrn
-from datahub.utilities.urns.urn import Urn
 
 
 class DataJobPatchBuilder(MetadataPatchProposal):
@@ -45,43 +42,6 @@ def __init__(
         )
         self.ownership_patch_helper = OwnershipPatchHelper(self)
 
-    def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass:
-        """
-        Creates an AuditStampClass instance with the current timestamp and other default values.
-
-        Args:
-            message: The message associated with the audit stamp (optional).
-
-        Returns:
-            An instance of AuditStampClass.
-        """
-        return AuditStampClass(
-            time=int(time.time() * 1000.0),
-            actor="urn:li:corpuser:datahub",
-            message=message,
-        )
-
-    def _ensure_urn_type(
-        self, entity_type: str, edges: List[Edge], context: str
-    ) -> None:
-        """
-        Ensures that the destination URNs in the given edges have the specified entity type.
-
-        Args:
-            entity_type: The entity type to check against.
-            edges: A list of Edge objects.
-            context: The context or description of the operation.
-
-        Raises:
-            ValueError: If any of the destination URNs is not of the specified entity type.
-        """
-        for e in edges:
-            urn = Urn.create_from_string(e.destinationUrn)
-            if not urn.get_type() == entity_type:
-                raise ValueError(
-                    f"{context}: {e.destinationUrn} is not of type {entity_type}"
-                )
-
     def add_owner(self, owner: Owner) -> "DataJobPatchBuilder":
         """
         Adds an owner to the DataJobPatchBuilder.
@@ -142,7 +102,7 @@ def add_input_datajob(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilde
 
         Notes:
             If `input` is an Edge object, it is used directly. If `input` is a Urn object or string,
-            it is converted to an Edge object and added with default audit stamps.
+            it is converted to an Edge object and added without any audit stamps.
         """
         if isinstance(input, Edge):
             input_urn: str = input.destinationUrn
@@ -154,8 +114,6 @@ def add_input_datajob(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilde
 
             input_edge = Edge(
                 destinationUrn=input_urn,
-                created=self._mint_auditstamp(),
-                lastModified=self._mint_auditstamp(),
             )
 
         self._ensure_urn_type("dataJob", [input_edge], "add_input_datajob")
@@ -225,7 +183,7 @@ def add_input_dataset(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilde
 
         Notes:
             If `input` is an Edge object, it is used directly. If `input` is a Urn object or string,
-            it is converted to an Edge object and added with default audit stamps.
+            it is converted to an Edge object and added without any audit stamps.
         """
         if isinstance(input, Edge):
             input_urn: str = input.destinationUrn
@@ -237,8 +195,6 @@ def add_input_dataset(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilde
 
             input_edge = Edge(
                 destinationUrn=input_urn,
-                created=self._mint_auditstamp(),
-                lastModified=self._mint_auditstamp(),
             )
 
         self._ensure_urn_type("dataset", [input_edge], "add_input_dataset")
@@ -310,7 +266,7 @@ def add_output_dataset(
 
         Notes:
             If `output` is an Edge object, it is used directly. If `output` is a Urn object or string,
-            it is converted to an Edge object and added with default audit stamps.
+            it is converted to an Edge object and added without any audit stamps.
         """
         if isinstance(output, Edge):
             output_urn: str = output.destinationUrn
@@ -322,15 +278,13 @@ def add_output_dataset(
 
             output_edge = Edge(
                 destinationUrn=output_urn,
-                created=self._mint_auditstamp(),
-                lastModified=self._mint_auditstamp(),
             )
 
         self._ensure_urn_type("dataset", [output_edge], "add_output_dataset")
         self._add_patch(
             DataJobInputOutput.ASPECT_NAME,
             "add",
-            path=f"/outputDatasetEdges/{self.quote(str(output))}",
+            path=f"/outputDatasetEdges/{self.quote(output_urn)}",
             value=output_edge,
         )
         return self
@@ -392,9 +346,7 @@ def add_input_dataset_field(self, input: Union[Urn, str]) -> "DataJobPatchBuilde
             ValueError: If the input is not a Schema Field urn.
         """
         input_urn = str(input)
-        urn = Urn.create_from_string(input_urn)
-        if not urn.get_type() == "schemaField":
-            raise ValueError(f"Input {input} is not a Schema Field urn")
+        assert SchemaFieldUrn.from_string(input_urn)
 
         self._add_patch(
             DataJobInputOutput.ASPECT_NAME,
@@ -466,9 +418,7 @@ def add_output_dataset_field(
             ValueError: If the output is not a Schema Field urn.
         """
         output_urn = str(output)
-        urn = Urn.create_from_string(output_urn)
-        if not urn.get_type() == "schemaField":
-            raise ValueError(f"Input {output} is not a Schema Field urn")
+        assert SchemaFieldUrn.from_string(output_urn)
 
         self._add_patch(
             DataJobInputOutput.ASPECT_NAME,
diff --git a/metadata-ingestion/src/datahub/specific/dataproduct.py b/metadata-ingestion/src/datahub/specific/dataproduct.py
index 6b7e695b4d57e..f9830a4b23df0 100644
--- a/metadata-ingestion/src/datahub/specific/dataproduct.py
+++ b/metadata-ingestion/src/datahub/specific/dataproduct.py
@@ -131,7 +131,7 @@ def add_asset(self, asset_urn: str) -> "DataProductPatchBuilder":
         self._add_patch(
             DataProductProperties.ASPECT_NAME,
             "add",
-            path=f"/assets/{asset_urn}",
+            path=f"/assets/{self.quote(asset_urn)}",
             value=DataProductAssociation(destinationUrn=asset_urn),
         )
         return self
@@ -140,7 +140,7 @@ def remove_asset(self, asset_urn: str) -> "DataProductPatchBuilder":
         self._add_patch(
             DataProductProperties.ASPECT_NAME,
             "remove",
-            path=f"/assets/{asset_urn}",
+            path=f"/assets/{self.quote(asset_urn)}",
             value={},
         )
         return self
diff --git a/metadata-ingestion/src/datahub/specific/dataset.py b/metadata-ingestion/src/datahub/specific/dataset.py
index 9dd2616078f08..b171dc4cc2939 100644
--- a/metadata-ingestion/src/datahub/specific/dataset.py
+++ b/metadata-ingestion/src/datahub/specific/dataset.py
@@ -13,7 +13,7 @@
     KafkaAuditHeaderClass,
     OwnerClass as Owner,
     OwnershipTypeClass,
-    SchemaMetadataClass as SchemaMetadata,
+    SchemaMetadataClass,
     SystemMetadataClass,
     TagAssociationClass as Tag,
     UpstreamClass as Upstream,
@@ -40,7 +40,7 @@ def __init__(
         self.aspect_name = (
             EditableSchemaMetadata.ASPECT_NAME
             if editable
-            else SchemaMetadata.ASPECT_NAME
+            else SchemaMetadataClass.ASPECT_NAME
         )
         self.aspect_field = "editableSchemaFieldInfo" if editable else "schemaFieldInfo"
 
diff --git a/metadata-ingestion/src/datahub/sql_parsing/_models.py b/metadata-ingestion/src/datahub/sql_parsing/_models.py
index d92d178b81cf4..d586e7d6d9045 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/_models.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/_models.py
@@ -42,6 +42,8 @@ def __lt__(self, other: "_FrozenModel") -> bool:
 
 
 class _TableName(_FrozenModel):
+    # TODO: Move this into the schema_resolver.py file.
+
     database: Optional[str] = None
     db_schema: Optional[str] = None
     table: str
diff --git a/metadata-ingestion/src/datahub/sql_parsing/_sqlglot_patch.py b/metadata-ingestion/src/datahub/sql_parsing/_sqlglot_patch.py
new file mode 100644
index 0000000000000..55f30b576b44e
--- /dev/null
+++ b/metadata-ingestion/src/datahub/sql_parsing/_sqlglot_patch.py
@@ -0,0 +1,215 @@
+import dataclasses
+import difflib
+import logging
+
+import patchy.api
+import sqlglot
+import sqlglot.expressions
+import sqlglot.lineage
+import sqlglot.optimizer.scope
+import sqlglot.optimizer.unnest_subqueries
+
+from datahub.utilities.is_pytest import is_pytest_running
+from datahub.utilities.unified_diff import apply_diff
+
+# This injects a few patches into sqlglot to add features and mitigate
+# some bugs and performance issues.
+# The diffs in this file should match the diffs declared in our fork.
+# https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main
+# For a diff-formatted view, see:
+# https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main.diff
+
+_DEBUG_PATCHER = is_pytest_running()
+logger = logging.getLogger(__name__)
+
+_apply_diff_subprocess = patchy.api._apply_patch
+
+
+def _new_apply_patch(source: str, patch_text: str, forwards: bool, name: str) -> str:
+    assert forwards, "Only forward patches are supported"
+
+    result = apply_diff(source, patch_text)
+
+    # TODO: When in testing mode, still run the subprocess and check that the
+    # results line up.
+    if _DEBUG_PATCHER:
+        result_subprocess = _apply_diff_subprocess(source, patch_text, forwards, name)
+        if result_subprocess != result:
+            logger.info("Results from subprocess and _apply_diff do not match")
+            logger.debug(f"Subprocess result:\n{result_subprocess}")
+            logger.debug(f"Our result:\n{result}")
+            diff = difflib.unified_diff(
+                result_subprocess.splitlines(), result.splitlines()
+            )
+            logger.debug("Diff:\n" + "\n".join(diff))
+            raise ValueError("Results from subprocess and _apply_diff do not match")
+
+    return result
+
+
+patchy.api._apply_patch = _new_apply_patch
+
+
+def _patch_deepcopy() -> None:
+    patchy.patch(
+        sqlglot.expressions.Expression.__deepcopy__,
+        """\
+@@ -1,4 +1,7 @@ def meta(self) -> t.Dict[str, t.Any]:
+ def __deepcopy__(self, memo):
++    import datahub.utilities.cooperative_timeout
++    datahub.utilities.cooperative_timeout.cooperate()
++
+     root = self.__class__()
+     stack = [(self, root)]
+""",
+    )
+
+
+def _patch_scope_traverse() -> None:
+    # Circular scope dependencies can happen in somewhat specific circumstances
+    # due to our usage of sqlglot.
+    # See https://github.com/tobymao/sqlglot/pull/4244
+    patchy.patch(
+        sqlglot.optimizer.scope.Scope.traverse,
+        """\
+@@ -5,9 +5,16 @@ def traverse(self):
+         Scope: scope instances in depth-first-search post-order
+     \"""
+     stack = [self]
++    seen_scopes = set()
+     result = []
+     while stack:
+         scope = stack.pop()
++
++        # Scopes aren't hashable, so we use id(scope) instead.
++        if id(scope) in seen_scopes:
++            raise OptimizeError(f"Scope {scope} has a circular scope dependency")
++        seen_scopes.add(id(scope))
++
+         result.append(scope)
+         stack.extend(
+             itertools.chain(
+""",
+    )
+
+
+def _patch_unnest_subqueries() -> None:
+    patchy.patch(
+        sqlglot.optimizer.unnest_subqueries.decorrelate,
+        """\
+@@ -261,16 +261,19 @@ def remove_aggs(node):
+         if key in group_by:
+             key.replace(nested)
+         elif isinstance(predicate, exp.EQ):
+-            parent_predicate = _replace(
+-                parent_predicate,
+-                f"({parent_predicate} AND ARRAY_CONTAINS({nested}, {column}))",
+-            )
++            if parent_predicate:
++                parent_predicate = _replace(
++                    parent_predicate,
++                    f"({parent_predicate} AND ARRAY_CONTAINS({nested}, {column}))",
++                )
+         else:
+             key.replace(exp.to_identifier("_x"))
+-            parent_predicate = _replace(
+-                parent_predicate,
+-                f"({parent_predicate} AND ARRAY_ANY({nested}, _x -> {predicate}))",
+-            )
++
++            if parent_predicate:
++                parent_predicate = _replace(
++                    parent_predicate,
++                    f"({parent_predicate} AND ARRAY_ANY({nested}, _x -> {predicate}))",
++                )
+""",
+    )
+
+
+def _patch_lineage() -> None:
+    # Add the "subfield" attribute to sqlglot.lineage.Node.
+    # With dataclasses, the easiest way to do this is with inheritance.
+    # Unfortunately, mypy won't pick up on the new field, so we need to
+    # use type ignores everywhere we use subfield.
+    @dataclasses.dataclass(frozen=True)
+    class Node(sqlglot.lineage.Node):
+        subfield: str = ""
+
+    sqlglot.lineage.Node = Node  # type: ignore
+
+    patchy.patch(
+        sqlglot.lineage.lineage,
+        """\
+@@ -12,7 +12,8 @@ def lineage(
+     \"""
+
+     expression = maybe_parse(sql, dialect=dialect)
+-    column = normalize_identifiers.normalize_identifiers(column, dialect=dialect).name
++    # column = normalize_identifiers.normalize_identifiers(column, dialect=dialect).name
++    assert isinstance(column, str)
+
+     if sources:
+         expression = exp.expand(
+""",
+    )
+
+    patchy.patch(
+        sqlglot.lineage.to_node,
+        """\
+@@ -235,11 +237,12 @@ def to_node(
+             )
+
+     # Find all columns that went into creating this one to list their lineage nodes.
+-    source_columns = set(find_all_in_scope(select, exp.Column))
++    source_columns = list(find_all_in_scope(select, exp.Column))
+
+-    # If the source is a UDTF find columns used in the UTDF to generate the table
++    # If the source is a UDTF find columns used in the UDTF to generate the table
++    source = scope.expression
+     if isinstance(source, exp.UDTF):
+-        source_columns |= set(source.find_all(exp.Column))
++        source_columns += list(source.find_all(exp.Column))
+         derived_tables = [
+             source.expression.parent
+             for source in scope.sources.values()
+@@ -254,6 +257,7 @@ def to_node(
+         if dt.comments and dt.comments[0].startswith("source: ")
+     }
+
++    c: exp.Column
+     for c in source_columns:
+         table = c.table
+         source = scope.sources.get(table)
+@@ -281,8 +285,21 @@ def to_node(
+             # it means this column's lineage is unknown. This can happen if the definition of a source used in a query
+             # is not passed into the `sources` map.
+             source = source or exp.Placeholder()
++
++            subfields = []
++            field: exp.Expression = c
++            while isinstance(field.parent, exp.Dot):
++                field = field.parent
++                subfields.append(field.name)
++            subfield = ".".join(subfields)
++
+             node.downstream.append(
+-                Node(name=c.sql(comments=False), source=source, expression=source)
++                Node(
++                    name=c.sql(comments=False),
++                    source=source,
++                    expression=source,
++                    subfield=subfield,
++                )
+             )
+
+     return node
+""",
+    )
+
+
+_patch_deepcopy()
+_patch_scope_traverse()
+_patch_unnest_subqueries()
+_patch_lineage()
+
+SQLGLOT_PATCHED = True
diff --git a/metadata-ingestion/src/datahub/sql_parsing/datajob.py b/metadata-ingestion/src/datahub/sql_parsing/datajob.py
new file mode 100644
index 0000000000000..215b207c3dcf5
--- /dev/null
+++ b/metadata-ingestion/src/datahub/sql_parsing/datajob.py
@@ -0,0 +1,50 @@
+import logging
+from typing import Iterable, List, Optional
+
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.schema_classes import (
+    DataJobInputOutputClass,
+    FineGrainedLineageClass,
+    UpstreamLineageClass,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def to_datajob_input_output(
+    *, mcps: Iterable[MetadataChangeProposalWrapper], ignore_extra_mcps: bool = True
+) -> Optional[DataJobInputOutputClass]:
+    inputDatasets: List[str] = []
+    outputDatasets: List[str] = []
+    fineGrainedLineages: List[FineGrainedLineageClass] = []
+    for mcp in mcps:
+        # TODO: Represent simple write operations without lineage as outputDatasets.
+
+        upstream_lineage = mcp.as_workunit().get_aspect_of_type(UpstreamLineageClass)
+        if upstream_lineage is not None:
+            if mcp.entityUrn and mcp.entityUrn not in outputDatasets:
+                outputDatasets.append(mcp.entityUrn)
+
+            for upstream in upstream_lineage.upstreams:
+                if upstream.dataset not in inputDatasets:
+                    inputDatasets.append(upstream.dataset)
+
+            if upstream_lineage.fineGrainedLineages:
+                for fineGrainedLineage in upstream_lineage.fineGrainedLineages:
+                    fineGrainedLineages.append(fineGrainedLineage)
+
+        elif ignore_extra_mcps:
+            pass
+        else:
+            raise ValueError(
+                f"Expected an upstreamLineage aspect, got {mcp.aspectName} for {mcp.entityUrn}"
+            )
+
+    if not inputDatasets and not outputDatasets:
+        return None
+
+    return DataJobInputOutputClass(
+        inputDatasets=inputDatasets,
+        outputDatasets=outputDatasets,
+        fineGrainedLineages=fineGrainedLineages,
+    )
diff --git a/metadata-ingestion/src/datahub/sql_parsing/query_types.py b/metadata-ingestion/src/datahub/sql_parsing/query_types.py
index 2acad19418c11..802fb3e993f42 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/query_types.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/query_types.py
@@ -14,7 +14,16 @@ def _is_temp_table(table: sqlglot.exp.Table, dialect: sqlglot.Dialect) -> bool:
     identifier: sqlglot.exp.Identifier = table.this
 
     return identifier.args.get("temporary") or (
-        is_dialect_instance(dialect, "redshift") and identifier.name.startswith("#")
+        # These dialects use # as a prefix for temp tables.
+        is_dialect_instance(
+            dialect,
+            [
+                "redshift",
+                "mssql",
+                # sybase is another one, but we don't support that dialect yet.
+            ],
+        )
+        and identifier.name.startswith("#")
     )
 
 
diff --git a/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py b/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py
index c358bee6daae8..6aa10381a883e 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py
@@ -13,7 +13,7 @@
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier
 from datahub.metadata.schema_classes import SchemaFieldClass, SchemaMetadataClass
 from datahub.metadata.urns import DataPlatformUrn
-from datahub.sql_parsing._models import _TableName
+from datahub.sql_parsing._models import _TableName as _TableName  # noqa: I250
 from datahub.sql_parsing.sql_parsing_common import PLATFORMS_WITH_CASE_SENSITIVE_TABLES
 from datahub.utilities.file_backed_collections import ConnectionWrapper, FileBackedDict
 from datahub.utilities.urns.field_paths import get_simple_field_path_from_v2_field_path
@@ -90,7 +90,9 @@ def schema_count(self) -> int:
             )[0][0]
         )
 
-    def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str:
+    def get_urn_for_table(
+        self, table: _TableName, lower: bool = False, mixed: bool = False
+    ) -> str:
         # TODO: Validate that this is the correct 2/3 layer hierarchy for the platform.
 
         table_name = ".".join(
@@ -101,7 +103,10 @@ def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str:
 
         if lower:
             table_name = table_name.lower()
-            platform_instance = platform_instance.lower() if platform_instance else None
+            if not mixed:
+                platform_instance = (
+                    platform_instance.lower() if platform_instance else None
+                )
 
         if self.platform == "bigquery":
             # Normalize shard numbers and other BigQuery weirdness.
@@ -118,6 +123,13 @@ def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str:
         )
         return urn
 
+    def resolve_urn(self, urn: str) -> Tuple[str, Optional[SchemaInfo]]:
+        schema_info = self._resolve_schema_info(urn)
+        if schema_info:
+            return urn, schema_info
+
+        return urn, None
+
     def resolve_table(self, table: _TableName) -> Tuple[str, Optional[SchemaInfo]]:
         urn = self.get_urn_for_table(table)
 
@@ -131,6 +143,20 @@ def resolve_table(self, table: _TableName) -> Tuple[str, Optional[SchemaInfo]]:
             if schema_info:
                 return urn_lower, schema_info
 
+        # Our treatment of platform instances when lowercasing urns
+        # is inconsistent. In some places (e.g. Snowflake), we lowercase
+        # the table names but not the platform instance. In other places
+        # (e.g. Databricks), we lowercase everything because it happens
+        # via the automatic lowercasing helper.
+        # See https://github.com/datahub-project/datahub/pull/8928.
+        # While we have this sort of inconsistency, we should also
+        # check the mixed case urn, as a last resort.
+        urn_mixed = self.get_urn_for_table(table, lower=True, mixed=True)
+        if urn_mixed not in {urn, urn_lower}:
+            schema_info = self._resolve_schema_info(urn_mixed)
+            if schema_info:
+                return urn_mixed, schema_info
+
         if self._prefers_urn_lower():
             return urn_lower, None
         else:
@@ -274,3 +300,19 @@ def _convert_schema_field_list_to_info(
 
 def _convert_schema_aspect_to_info(schema_metadata: SchemaMetadataClass) -> SchemaInfo:
     return _convert_schema_field_list_to_info(schema_metadata.fields)
+
+
+def match_columns_to_schema(
+    schema_info: SchemaInfo, input_columns: List[str]
+) -> List[str]:
+    column_from_gms: List[str] = list(schema_info.keys())  # list() to silent lint
+
+    gms_column_map: Dict[str, str] = {
+        column.lower(): column for column in column_from_gms
+    }
+
+    output_columns: List[str] = [
+        gms_column_map.get(column.lower(), column) for column in input_columns
+    ]
+
+    return output_columns
diff --git a/metadata-ingestion/src/datahub/sql_parsing/split_statements.py b/metadata-ingestion/src/datahub/sql_parsing/split_statements.py
new file mode 100644
index 0000000000000..42dda4e62158b
--- /dev/null
+++ b/metadata-ingestion/src/datahub/sql_parsing/split_statements.py
@@ -0,0 +1,163 @@
+import re
+from enum import Enum
+from typing import Generator, List, Tuple
+
+CONTROL_FLOW_KEYWORDS = [
+    "GO",
+    r"BEGIN\w+TRY",
+    r"BEGIN\w+CATCH",
+    "BEGIN",
+    r"END\w+TRY",
+    r"END\w+CATCH",
+    "END",
+]
+
+# There's an exception to this rule, which is when the statement
+# is preceeded by a CTE.
+FORCE_NEW_STATEMENT_KEYWORDS = [
+    # SELECT is used inside queries as well, so we can't include it here.
+    "INSERT",
+    "UPDATE",
+    "DELETE",
+    "MERGE",
+]
+
+
+class ParserState(Enum):
+    NORMAL = 1
+    STRING = 2
+    COMMENT = 3
+    MULTILINE_COMMENT = 4
+
+
+def _is_keyword_at_position(sql: str, pos: int, keyword: str) -> bool:
+    """
+    Check if a keyword exists at the given position using regex word boundaries.
+    """
+    if pos + len(keyword) > len(sql):
+        return False
+
+    # If we're not at a word boundary, we can't generate a keyword.
+    if pos > 0 and not (
+        bool(re.match(r"\w\W", sql[pos - 1 : pos + 1]))
+        or bool(re.match(r"\W\w", sql[pos - 1 : pos + 1]))
+    ):
+        return False
+
+    pattern = rf"^{re.escape(keyword)}\b"
+    match = re.match(pattern, sql[pos:], re.IGNORECASE)
+    return bool(match)
+
+
+def _look_ahead_for_keywords(
+    sql: str, pos: int, keywords: List[str]
+) -> Tuple[bool, str, int]:
+    """
+    Look ahead for SQL keywords at the current position.
+    """
+
+    for keyword in keywords:
+        if _is_keyword_at_position(sql, pos, keyword):
+            return True, keyword, len(keyword)
+    return False, "", 0
+
+
+def split_statements(sql: str) -> Generator[str, None, None]:
+    """
+    Split T-SQL code into individual statements, handling various SQL constructs.
+    """
+    if not sql or not sql.strip():
+        return
+
+    current_statement: List[str] = []
+    state = ParserState.NORMAL
+    i = 0
+
+    def yield_if_complete() -> Generator[str, None, None]:
+        statement = "".join(current_statement).strip()
+        if statement:
+            yield statement
+            current_statement.clear()
+
+    prev_real_char = "\0"  # the most recent non-whitespace, non-comment character
+    while i < len(sql):
+        c = sql[i]
+        next_char = sql[i + 1] if i < len(sql) - 1 else "\0"
+
+        if state == ParserState.NORMAL:
+            if c == "'":
+                state = ParserState.STRING
+                current_statement.append(c)
+                prev_real_char = c
+            elif c == "-" and next_char == "-":
+                state = ParserState.COMMENT
+                current_statement.append(c)
+                current_statement.append(next_char)
+                i += 1
+            elif c == "/" and next_char == "*":
+                state = ParserState.MULTILINE_COMMENT
+                current_statement.append(c)
+                current_statement.append(next_char)
+                i += 1
+            else:
+                most_recent_real_char = prev_real_char
+                if not c.isspace():
+                    prev_real_char = c
+
+                is_control_keyword, keyword, keyword_len = _look_ahead_for_keywords(
+                    sql, i, keywords=CONTROL_FLOW_KEYWORDS
+                )
+                if is_control_keyword:
+                    # Yield current statement if any
+                    yield from yield_if_complete()
+                    # Yield keyword as its own statement
+                    yield keyword
+                    i += keyword_len
+                    continue
+
+                (
+                    is_force_new_statement_keyword,
+                    keyword,
+                    keyword_len,
+                ) = _look_ahead_for_keywords(
+                    sql, i, keywords=FORCE_NEW_STATEMENT_KEYWORDS
+                )
+                if (
+                    is_force_new_statement_keyword and most_recent_real_char != ")"
+                ):  # usually we'd have a close paren that closes a CTE
+                    # Force termination of current statement
+                    yield from yield_if_complete()
+
+                    current_statement.append(keyword)
+                    i += keyword_len
+                    continue
+
+                elif c == ";":
+                    yield from yield_if_complete()
+                else:
+                    current_statement.append(c)
+
+        elif state == ParserState.STRING:
+            current_statement.append(c)
+            if c == "'" and next_char == "'":
+                current_statement.append(next_char)
+                i += 1
+            elif c == "'":
+                state = ParserState.NORMAL
+
+        elif state == ParserState.COMMENT:
+            current_statement.append(c)
+            if c == "\n":
+                state = ParserState.NORMAL
+
+        elif state == ParserState.MULTILINE_COMMENT:
+            current_statement.append(c)
+            if c == "*" and next_char == "/":
+                current_statement.append(next_char)
+                i += 1
+                state = ParserState.NORMAL
+
+        i += 1
+
+    # Handle the last statement
+    yield from yield_if_complete()
diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
index 52934f9f72a70..f81eb291e89e1 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
@@ -134,7 +134,7 @@ class QueryMetadata:
 
     upstreams: List[UrnStr]  # this is direct upstreams, which may be temp tables
     column_lineage: List[ColumnLineageInfo]
-    column_usage: Dict[UrnStr, Set[UrnStr]]
+    column_usage: Dict[UrnStr, Set[UrnStr]]  # TODO: Change to an OrderedSet
     confidence_score: float
 
     used_temp_tables: bool = True
@@ -174,6 +174,28 @@ class KnownLineageMapping:
     lineage_type: str = models.DatasetLineageTypeClass.COPY
 
 
+@dataclasses.dataclass
+class TableRename:
+    original_urn: UrnStr
+    new_urn: UrnStr
+    query: Optional[str] = None
+    session_id: str = _MISSING_SESSION_ID
+    timestamp: Optional[datetime] = None
+
+
+@dataclasses.dataclass
+class TableSwap:
+    urn1: UrnStr
+    urn2: UrnStr
+    query: Optional[str] = None
+    session_id: str = _MISSING_SESSION_ID
+    timestamp: Optional[datetime] = None
+
+    def id(self) -> str:
+        # TableSwap(A,B) is same as TableSwap(B,A)
+        return str(hash(frozenset([self.urn1, self.urn2])))
+
+
 @dataclasses.dataclass
 class PreparsedQuery:
     # If not provided, we will generate one using the fast fingerprint generator.
@@ -237,6 +259,7 @@ class SqlAggregatorReport(Report):
     num_preparsed_queries: int = 0
     num_known_mapping_lineage: int = 0
     num_table_renames: int = 0
+    num_table_swaps: int = 0
 
     # Temp tables.
     num_temp_sessions: Optional[int] = None
@@ -275,6 +298,8 @@ class SqlAggregatorReport(Report):
     tool_meta_report: Optional[ToolMetaExtractorReport] = None
 
     def compute_stats(self) -> None:
+        if self._aggregator._closed:
+            return
         self.schema_resolver_count = self._aggregator._schema_resolver.schema_count()
         self.num_unique_query_fingerprints = len(self._aggregator._query_map)
 
@@ -345,6 +370,7 @@ def __init__(
 
         # The exit stack helps ensure that we close all the resources we open.
         self._exit_stack = contextlib.ExitStack()
+        self._closed: bool = False
 
         # Set up the schema resolver.
         self._schema_resolver: SchemaResolver
@@ -420,8 +446,7 @@ def __init__(
 
         # Map of session ID -> {temp table name -> query id}
         # Needs to use the query_map to find the info about the query.
-        # This assumes that a temp table is created at most once per session.
-        self._temp_lineage_map = FileBackedDict[Dict[UrnStr, QueryId]](
+        self._temp_lineage_map = FileBackedDict[Dict[UrnStr, OrderedSet[QueryId]]](
             shared_connection=self._shared_connection, tablename="temp_lineage_map"
         )
         self._exit_stack.push(self._temp_lineage_map)
@@ -439,6 +464,12 @@ def __init__(
         )
         self._exit_stack.push(self._table_renames)
 
+        # Map of table swaps, from unique swap id to TableSwap
+        self._table_swaps = FileBackedDict[TableSwap](
+            shared_connection=self._shared_connection, tablename="table_swaps"
+        )
+        self._exit_stack.push(self._table_swaps)
+
         # Usage aggregator. This will only be initialized if usage statistics are enabled.
         # TODO: Replace with FileBackedDict.
         # TODO: The BaseUsageConfig class is much too broad for our purposes, and has a number of
@@ -456,12 +487,16 @@ def __init__(
                 shared_connection=self._shared_connection,
                 tablename="query_usage_counts",
             )
+            self._exit_stack.push(self._query_usage_counts)
 
         # Tool Extractor
-        self._tool_meta_extractor = ToolMetaExtractor()
+        self._tool_meta_extractor = ToolMetaExtractor.create(graph)
         self.report.tool_meta_report = self._tool_meta_extractor.report
 
     def close(self) -> None:
+        # Compute stats once before closing connections
+        self.report.compute_stats()
+        self._closed = True
         self._exit_stack.close()
 
     @property
@@ -502,8 +537,13 @@ def _maybe_format_query(self, query: str) -> str:
         return query
 
     @functools.lru_cache(maxsize=128)
-    def _name_from_urn(self, urn: UrnStr) -> str:
-        name = DatasetUrn.from_string(urn).name
+    def _name_from_urn(self, urn: UrnStr) -> Optional[str]:
+        urn_obj = DatasetUrn.from_string(urn)
+        if urn_obj.platform != self.platform.urn():
+            # If this is external (e.g. s3), we don't know the name.
+            return None
+
+        name = urn_obj.name
         if (
             platform_instance := self._schema_resolver.platform_instance
         ) and name.startswith(platform_instance):
@@ -514,19 +554,32 @@ def _name_from_urn(self, urn: UrnStr) -> str:
     def is_temp_table(self, urn: UrnStr) -> bool:
         if self._is_temp_table is None:
             return False
-        return self._is_temp_table(self._name_from_urn(urn))
+        name = self._name_from_urn(urn)
+        if name is None:
+            # External tables are not temp tables.
+            return False
+        return self._is_temp_table(name)
 
-    def is_allowed_table(self, urn: UrnStr) -> bool:
+    def is_allowed_table(self, urn: UrnStr, allow_external: bool = True) -> bool:
         if self.is_temp_table(urn):
             return False
         if self._is_allowed_table is None:
             return True
-        return self._is_allowed_table(self._name_from_urn(urn))
+        name = self._name_from_urn(urn)
+        if name is None:
+            # Treat external tables specially.
+            return allow_external
+        return self._is_allowed_table(name)
 
     def add(
         self,
         item: Union[
-            KnownQueryLineageInfo, KnownLineageMapping, PreparsedQuery, ObservedQuery
+            KnownQueryLineageInfo,
+            KnownLineageMapping,
+            PreparsedQuery,
+            ObservedQuery,
+            TableRename,
+            TableSwap,
         ],
     ) -> None:
         if isinstance(item, KnownQueryLineageInfo):
@@ -537,6 +590,10 @@ def add(
             self.add_preparsed_query(item)
         elif isinstance(item, ObservedQuery):
             self.add_observed_query(item)
+        elif isinstance(item, TableRename):
+            self.add_table_rename(item)
+        elif isinstance(item, TableSwap):
+            self.add_table_swap(item)
         else:
             raise ValueError(f"Cannot add unknown item type: {type(item)}")
 
@@ -613,26 +670,20 @@ def add_known_lineage_mapping(
             upstream_urn: The upstream dataset URN.
             downstream_urn: The downstream dataset URN.
         """
-
+        logger.debug(
+            f"Adding lineage to the map, downstream: {downstream_urn}, upstream: {upstream_urn}"
+        )
         self.report.num_known_mapping_lineage += 1
 
         # We generate a fake "query" object to hold the lineage.
         query_id = self._known_lineage_query_id()
 
         # Generate CLL if schema of downstream is known
-        column_lineage: List[ColumnLineageInfo] = []
-        if self._schema_resolver.has_urn(downstream_urn):
-            schema = self._schema_resolver._resolve_schema_info(downstream_urn)
-            if schema:
-                column_lineage = [
-                    ColumnLineageInfo(
-                        downstream=DownstreamColumnRef(
-                            table=downstream_urn, column=field_path
-                        ),
-                        upstreams=[ColumnRef(table=upstream_urn, column=field_path)],
-                    )
-                    for field_path in schema
-                ]
+        column_lineage: List[
+            ColumnLineageInfo
+        ] = self._generate_identity_column_lineage(
+            upstream_urn=upstream_urn, downstream_urn=downstream_urn
+        )
 
         # Register the query.
         self._add_to_query_map(
@@ -654,6 +705,25 @@ def add_known_lineage_mapping(
         # Register the lineage.
         self._lineage_map.for_mutation(downstream_urn, OrderedSet()).add(query_id)
 
+    def _generate_identity_column_lineage(
+        self, *, upstream_urn: UrnStr, downstream_urn: UrnStr
+    ) -> List[ColumnLineageInfo]:
+        column_lineage: List[ColumnLineageInfo] = []
+        if self._schema_resolver.has_urn(downstream_urn):
+            schema = self._schema_resolver._resolve_schema_info(downstream_urn)
+            if schema:
+                column_lineage = [
+                    ColumnLineageInfo(
+                        downstream=DownstreamColumnRef(
+                            table=downstream_urn, column=field_path
+                        ),
+                        upstreams=[ColumnRef(table=upstream_urn, column=field_path)],
+                    )
+                    for field_path in schema
+                ]
+
+        return column_lineage
+
     def add_view_definition(
         self,
         view_urn: Union[DatasetUrn, UrnStr],
@@ -692,7 +762,6 @@ def add_observed_query(
 
         This assumes that queries come in order of increasing timestamps.
         """
-
         self.report.num_observed_queries += 1
 
         # All queries with no session ID are assumed to be part of the same session.
@@ -761,7 +830,6 @@ def add_preparsed_query(
         session_has_temp_tables: bool = True,
         _is_internal: bool = False,
     ) -> None:
-
         # Adding tool specific metadata extraction here allows it
         # to work for both ObservedQuery and PreparsedQuery as
         # add_preparsed_query it used within add_observed_query.
@@ -795,7 +863,7 @@ def add_preparsed_query(
             upstream_fields = parsed.column_usage or {}
             for upstream_urn in parsed.upstreams:
                 # If the upstream table is a temp table or otherwise denied by filters, don't log usage for it.
-                if not self.is_allowed_table(upstream_urn) or (
+                if not self.is_allowed_table(upstream_urn, allow_external=False) or (
                     require_out_table_schema
                     and not self._schema_resolver.has_urn(upstream_urn)
                 ):
@@ -840,12 +908,6 @@ def add_preparsed_query(
             return
         out_table = parsed.downstream
 
-        # Handle table renames.
-        is_renamed_table = False
-        if out_table in self._table_renames:
-            out_table = self._table_renames[out_table]
-            is_renamed_table = True
-
         # Register the query's lineage.
         if (
             is_known_temp_table
@@ -853,15 +915,10 @@ def add_preparsed_query(
                 parsed.query_type.is_create()
                 and parsed.query_type_props.get("temporary")
             )
+            or self.is_temp_table(out_table)
             or (
-                not is_renamed_table
-                and (
-                    self.is_temp_table(out_table)
-                    or (
-                        require_out_table_schema
-                        and not self._schema_resolver.has_urn(out_table)
-                    )
-                )
+                require_out_table_schema
+                and not self._schema_resolver.has_urn(out_table)
             )
         ):
             # Infer the schema of the output table and track it for later.
@@ -869,9 +926,9 @@ def add_preparsed_query(
                 self._inferred_temp_schemas[query_fingerprint] = parsed.inferred_schema
 
             # Also track the lineage for the temp table, for merging purposes later.
-            self._temp_lineage_map.for_mutation(parsed.session_id, {})[
-                out_table
-            ] = query_fingerprint
+            self._temp_lineage_map.for_mutation(parsed.session_id, {}).setdefault(
+                out_table, OrderedSet()
+            ).add(query_fingerprint)
 
             # Also update schema resolver for missing session id
             if parsed.session_id == _MISSING_SESSION_ID and parsed.inferred_schema:
@@ -887,26 +944,93 @@ def add_preparsed_query(
 
     def add_table_rename(
         self,
-        original_urn: UrnStr,
-        new_urn: UrnStr,
+        table_rename: TableRename,
     ) -> None:
         """Add a table rename to the aggregator.
 
-        This will so that all _future_ observed queries that reference the original urn
-        will instead generate usage and lineage for the new urn.
+        This will make all observed queries that reference the original urn
+        will instead generate lineage for the new urn.
+        """
 
-        Currently, this does not affect any queries that have already been observed.
-        TODO: Add a mechanism to update the lineage for queries that have already been observed.
+        self.report.num_table_renames += 1
+
+        # This will not work if the table is renamed multiple times.
+        self._table_renames[table_rename.original_urn] = table_rename.new_urn
+
+        original_table = self._name_from_urn(table_rename.original_urn)
+        new_table = self._name_from_urn(table_rename.new_urn)
+
+        self.add_preparsed_query(
+            PreparsedQuery(
+                query_id=None,
+                query_text=table_rename.query
+                or f"--Datahub generated query text--\n"
+                f"alter table {original_table} rename to {new_table}",
+                upstreams=[table_rename.original_urn],
+                downstream=table_rename.new_urn,
+                column_lineage=self._generate_identity_column_lineage(
+                    downstream_urn=table_rename.new_urn,
+                    upstream_urn=table_rename.original_urn,
+                ),
+                session_id=table_rename.session_id,
+                timestamp=table_rename.timestamp,
+            )
+        )
+
+    def add_table_swap(self, table_swap: TableSwap) -> None:
+        """Add a table swap to the aggregator.
 
         Args:
-            original_urn: The original dataset URN.
-            new_urn: The new dataset URN.
+            table_swap.urn1, table_swap.urn2: The dataset URNs to swap.
         """
 
-        self.report.num_table_renames += 1
+        if table_swap.id() in self._table_swaps:
+            # We have already processed this table swap once
+            return
 
-        # This will not work if the table is renamed multiple times.
-        self._table_renames[original_urn] = new_urn
+        self.report.num_table_swaps += 1
+        self._table_swaps[table_swap.id()] = table_swap
+        table1 = self._name_from_urn(table_swap.urn1)
+        table2 = self._name_from_urn(table_swap.urn2)
+
+        # NOTE: Both queries are different on purpose. Currently, we can not
+        # store (A->B) and (B->A) lineage against same query.
+
+        # NOTE: we do not store upstreams for temp table on purpose, as that would
+        # otherwise overwrite original upstream query of temp table because
+        # currently a temporay table can have only one upstream query.
+
+        if not self.is_temp_table(table_swap.urn2):
+            self.add_preparsed_query(
+                PreparsedQuery(
+                    query_id=None,
+                    query_text=f"--Datahub generated query text--"
+                    f"\nalter table {table1} swap with {table2}",
+                    upstreams=[table_swap.urn1],
+                    downstream=table_swap.urn2,
+                    column_lineage=self._generate_identity_column_lineage(
+                        upstream_urn=table_swap.urn1, downstream_urn=table_swap.urn2
+                    ),
+                    session_id=table_swap.session_id,
+                    timestamp=table_swap.timestamp,
+                )
+            )
+
+        if not self.is_temp_table(table_swap.urn1):
+            self.add_preparsed_query(
+                PreparsedQuery(
+                    query_id=None,
+                    query_text=f"--Datahub generated query text--\n"
+                    f"alter table {table2} swap with {table1}",
+                    upstreams=[table_swap.urn2],
+                    downstream=table_swap.urn1,
+                    column_lineage=self._generate_identity_column_lineage(
+                        upstream_urn=table_swap.urn2, downstream_urn=table_swap.urn1
+                    ),
+                    session_id=table_swap.session_id,
+                    timestamp=table_swap.timestamp,
+                )
+            )
 
     def _make_schema_resolver_for_session(
         self, session_id: str
@@ -916,10 +1040,13 @@ def _make_schema_resolver_for_session(
             schema_resolver = self._missing_session_schema_resolver
         elif session_id in self._temp_lineage_map:
             temp_table_schemas: Dict[str, Optional[List[models.SchemaFieldClass]]] = {}
-            for temp_table_urn, query_id in self._temp_lineage_map[session_id].items():
-                temp_table_schemas[temp_table_urn] = self._inferred_temp_schemas.get(
-                    query_id
-                )
+            for temp_table_urn, query_ids in self._temp_lineage_map[session_id].items():
+                for query_id in query_ids:
+                    temp_table_schemas[
+                        temp_table_urn
+                    ] = self._inferred_temp_schemas.get(query_id)
+                    if temp_table_schemas:
+                        break
 
             if temp_table_schemas:
                 schema_resolver = self._schema_resolver.with_temp_tables(
@@ -1256,8 +1383,7 @@ def _query_urn(cls, query_id: QueryId) -> str:
         return QueryUrn(query_id).urn()
 
     @classmethod
-    def _composite_query_id(cls, composed_of_queries: Iterable[QueryId]) -> str:
-        composed_of_queries = list(composed_of_queries)
+    def _composite_query_id(cls, composed_of_queries: List[QueryId]) -> str:
         combined = json.dumps(composed_of_queries)
         return f"composite_{generate_hash(combined)}"
 
@@ -1310,7 +1436,7 @@ def _gen_query(
         for upstream in query.upstreams:
             query_subject_urns.add(upstream)
             if self.generate_query_subject_fields:
-                for column in query.column_usage.get(upstream, []):
+                for column in sorted(query.column_usage.get(upstream, [])):
                     query_subject_urns.add(
                         builder.make_schema_field_urn(upstream, column)
                     )
@@ -1413,6 +1539,13 @@ class QueryLineageInfo:
             column_lineage: List[ColumnLineageInfo]
             confidence_score: float
 
+            def _merge_lineage_from(self, other_query: "QueryLineageInfo") -> None:
+                self.upstreams += other_query.upstreams
+                self.column_lineage += other_query.column_lineage
+                self.confidence_score = min(
+                    self.confidence_score, other_query.confidence_score
+                )
+
         def _recurse_into_query(
             query: QueryMetadata, recursion_path: List[QueryId]
         ) -> QueryLineageInfo:
@@ -1429,15 +1562,27 @@ def _recurse_into_query(
             # Find all the temp tables that this query depends on.
             temp_upstream_queries: Dict[UrnStr, QueryLineageInfo] = {}
             for upstream in query.upstreams:
-                upstream_query_id = self._temp_lineage_map.get(session_id, {}).get(
+                upstream_query_ids = self._temp_lineage_map.get(session_id, {}).get(
                     upstream
                 )
-                if upstream_query_id:
-                    upstream_query = self._query_map.get(upstream_query_id)
-                    if upstream_query:
-                        temp_upstream_queries[upstream] = _recurse_into_query(
-                            upstream_query, recursion_path
-                        )
+                if upstream_query_ids:
+                    for upstream_query_id in upstream_query_ids:
+                        upstream_query = self._query_map.get(upstream_query_id)
+                        if (
+                            upstream_query
+                            and upstream_query.query_id not in composed_of_queries
+                        ):
+                            temp_query_lineage_info = _recurse_into_query(
+                                upstream_query, recursion_path
+                            )
+                            if upstream in temp_upstream_queries:
+                                temp_upstream_queries[upstream]._merge_lineage_from(
+                                    temp_query_lineage_info
+                                )
+                            else:
+                                temp_upstream_queries[
+                                    upstream
+                                ] = temp_query_lineage_info
 
             # Compute merged upstreams.
             new_upstreams = OrderedSet[UrnStr]()
@@ -1503,7 +1648,17 @@ def _recurse_into_query(
         # - Update the lineage info
         # - Update the query text to combine the queries
 
-        composite_query_id = self._composite_query_id(composed_of_queries)
+        ordered_queries = [
+            self._query_map[query_id] for query_id in reversed(composed_of_queries)
+        ]
+        if all(q.latest_timestamp for q in ordered_queries):
+            ordered_queries = sorted(
+                ordered_queries,
+                key=lambda query: make_ts_millis(query.latest_timestamp) or 0,
+            )
+        composite_query_id = self._composite_query_id(
+            [q.query_id for q in ordered_queries]
+        )
         composed_of_queries_truncated: LossyList[str] = LossyList()
         for query_id in composed_of_queries:
             composed_of_queries_truncated.append(query_id)
@@ -1512,10 +1667,7 @@ def _recurse_into_query(
         ] = composed_of_queries_truncated
 
         merged_query_text = ";\n\n".join(
-            [
-                self._query_map[query_id].formatted_query_string
-                for query_id in reversed(composed_of_queries)
-            ]
+            [q.formatted_query_string for q in ordered_queries]
         )
 
         resolved_query = dataclasses.replace(
diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py
index a1b850b56677a..ec7dbc8251b20 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py
@@ -21,6 +21,9 @@
     # See more below:
     # https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/n0ejgx4895bofnn14rlguktfx5r3.htm
     "teradata",
+    # For SQL server, the default collation rules mean that all identifiers (schema, table, column names)
+    # are case preserving but case insensitive.
+    "mssql",
 }
 DIALECTS_WITH_DEFAULT_UPPERCASE_COLS = {
     # In some dialects, column identifiers are effectively case insensitive
@@ -28,6 +31,9 @@
     # automatically lowercase unquoted identifiers.
     "snowflake",
 }
+assert DIALECTS_WITH_DEFAULT_UPPERCASE_COLS.issubset(
+    DIALECTS_WITH_CASE_INSENSITIVE_COLS
+)
 
 
 class QueryType(enum.Enum):
diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py
index 0806d0ec774fe..bf28ab0e7b229 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py
@@ -1,10 +1,11 @@
+from datahub.sql_parsing._sqlglot_patch import SQLGLOT_PATCHED
+
 import dataclasses
 import functools
-import itertools
 import logging
 import traceback
 from collections import defaultdict
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, TypeVar, Union
 
 import pydantic.dataclasses
 import sqlglot
@@ -14,6 +15,8 @@
 import sqlglot.optimizer.annotate_types
 import sqlglot.optimizer.optimizer
 import sqlglot.optimizer.qualify
+import sqlglot.optimizer.qualify_columns
+import sqlglot.optimizer.unnest_subqueries
 
 from datahub.cli.env_utils import get_boolean_env_variable
 from datahub.ingestion.graph.client import DataHubGraph
@@ -52,6 +55,8 @@
     cooperative_timeout,
 )
 
+assert SQLGLOT_PATCHED
+
 logger = logging.getLogger(__name__)
 
 Urn = str
@@ -61,26 +66,33 @@
     "SQL_LINEAGE_TIMEOUT_ENABLED", True
 )
 SQL_LINEAGE_TIMEOUT_SECONDS = 10
-
-
-RULES_BEFORE_TYPE_ANNOTATION: tuple = tuple(
-    filter(
-        lambda func: func.__name__
-        not in {
-            # Skip pushdown_predicates because it sometimes throws exceptions, and we
-            # don't actually need it for anything.
-            "pushdown_predicates",
-            # Skip normalize because it can sometimes be expensive.
-            "normalize",
-        },
-        itertools.takewhile(
-            lambda func: func != sqlglot.optimizer.annotate_types.annotate_types,
-            sqlglot.optimizer.optimizer.RULES,
-        ),
-    )
+SQL_PARSER_TRACE = get_boolean_env_variable("DATAHUB_SQL_PARSER_TRACE", False)
+
+
+# These rules are a subset of the rules in sqlglot.optimizer.optimizer.RULES.
+# If there's a change in their rules, we probably need to re-evaluate our list as well.
+assert len(sqlglot.optimizer.optimizer.RULES) == 14
+
+_OPTIMIZE_RULES = (
+    sqlglot.optimizer.optimizer.qualify,
+    # We need to enable this in order for annotate types to work.
+    sqlglot.optimizer.optimizer.pushdown_projections,
+    # sqlglot.optimizer.optimizer.normalize,  # causes perf issues
+    sqlglot.optimizer.optimizer.unnest_subqueries,
+    # sqlglot.optimizer.optimizer.pushdown_predicates,  # causes perf issues
+    # sqlglot.optimizer.optimizer.optimize_joins,
+    # sqlglot.optimizer.optimizer.eliminate_subqueries,
+    # sqlglot.optimizer.optimizer.merge_subqueries,
+    # sqlglot.optimizer.optimizer.eliminate_joins,
+    # sqlglot.optimizer.optimizer.eliminate_ctes,
+    sqlglot.optimizer.optimizer.quote_identifiers,
+    # These three are run separately or not run at all.
+    # sqlglot.optimizer.optimizer.annotate_types,
+    # sqlglot.optimizer.canonicalize.canonicalize,
+    # sqlglot.optimizer.simplify.simplify,
 )
-# Quick check that the rules were loaded correctly.
-assert 0 < len(RULES_BEFORE_TYPE_ANNOTATION) < len(sqlglot.optimizer.optimizer.RULES)
+
+_DEBUG_TYPE_ANNOTATIONS = False
 
 
 class _ColumnRef(_FrozenModel):
@@ -354,10 +366,11 @@ def _sqlglot_force_column_normalizer(
 
             return node
 
-        # logger.debug(
-        #     "Prior to case normalization sql %s",
-        #     statement.sql(pretty=True, dialect=dialect),
-        # )
+        if SQL_PARSER_TRACE:
+            logger.debug(
+                "Prior to case normalization sql %s",
+                statement.sql(pretty=True, dialect=dialect),
+            )
         statement = statement.transform(_sqlglot_force_column_normalizer, copy=False)
         # logger.debug(
         #     "Sql after casing normalization %s",
@@ -385,11 +398,12 @@ def _sqlglot_force_column_normalizer(
                 schema=sqlglot_db_schema,
                 qualify_columns=True,
                 validate_qualify_columns=False,
+                allow_partial_qualification=True,
                 identify=True,
                 # sqlglot calls the db -> schema -> table hierarchy "catalog", "db", "table".
                 catalog=default_db,
                 db=default_schema,
-                rules=RULES_BEFORE_TYPE_ANNOTATION,
+                rules=_OPTIMIZE_RULES,
             )
         except (sqlglot.errors.OptimizeError, ValueError) as e:
             raise SqlUnderstandingError(
@@ -408,6 +422,10 @@ def _sqlglot_force_column_normalizer(
         except (sqlglot.errors.OptimizeError, sqlglot.errors.ParseError) as e:
             # This is not a fatal error, so we can continue.
             logger.debug("sqlglot failed to annotate or parse types: %s", e)
+        if _DEBUG_TYPE_ANNOTATIONS and logger.isEnabledFor(logging.DEBUG):
+            logger.debug(
+                "Type annotated sql %s", statement.sql(pretty=True, dialect=dialect)
+            )
 
     return statement, _ColumnResolver(
         sqlglot_db_schema=sqlglot_db_schema,
@@ -546,7 +564,7 @@ def _select_statement_cll(  # noqa: C901
                 )
             )
 
-        # TODO: Also extract referenced columns (aka auxillary / non-SELECT lineage)
+        # TODO: Also extract referenced columns (aka auxiliary / non-SELECT lineage)
     except (sqlglot.errors.OptimizeError, ValueError, IndexError) as e:
         raise SqlUnderstandingError(
             f"sqlglot failed to compute some lineage: {e}"
@@ -656,7 +674,9 @@ def _get_direct_raw_col_upstreams(
             # Parse the column name out of the node name.
             # Sqlglot calls .sql(), so we have to do the inverse.
             normalized_col = sqlglot.parse_one(node.name).this.name
-            if node.subfield:
+            if hasattr(node, "subfield") and node.subfield:
+                # The hasattr check is necessary, since it lets us be compatible with
+                # sqlglot versions that don't have the subfield attribute.
                 normalized_col = f"{normalized_col}.{node.subfield}"
 
             direct_raw_col_upstreams.add(
@@ -855,6 +875,49 @@ def _translate_internal_column_lineage(
     )
 
 
+_StrOrNone = TypeVar("_StrOrNone", str, Optional[str])
+
+
+def _normalize_db_or_schema(
+    db_or_schema: _StrOrNone,
+    dialect: sqlglot.Dialect,
+) -> _StrOrNone:
+    if db_or_schema is None:
+        return None
+
+    # In snowflake, table identifiers must be uppercased to match sqlglot's behavior.
+    if is_dialect_instance(dialect, "snowflake"):
+        return db_or_schema.upper()
+
+    # In mssql, table identifiers must be lowercased.
+    elif is_dialect_instance(dialect, "mssql"):
+        return db_or_schema.lower()
+
+    return db_or_schema
+
+
+def _simplify_select_into(statement: sqlglot.exp.Expression) -> sqlglot.exp.Expression:
+    """
+    Check if the expression is a SELECT INTO statement. If so, converts it into a CTAS.
+    Other expressions are returned as-is.
+    """
+
+    if not (isinstance(statement, sqlglot.exp.Select) and statement.args.get("into")):
+        return statement
+
+    # Convert from SELECT <cols> INTO <out> <expr>
+    # to CREATE TABLE <out> AS SELECT <cols> <expr>
+    into_expr: sqlglot.exp.Into = statement.args["into"].pop()
+    into_table = into_expr.this
+
+    create = sqlglot.exp.Create(
+        this=into_table,
+        kind="TABLE",
+        expression=statement,
+    )
+    return create
+
+
 def _sqlglot_lineage_inner(
     sql: sqlglot.exp.ExpOrStr,
     schema_resolver: SchemaResolverInterface,
@@ -862,18 +925,14 @@ def _sqlglot_lineage_inner(
     default_schema: Optional[str] = None,
     default_dialect: Optional[str] = None,
 ) -> SqlParsingResult:
-
     if not default_dialect:
         dialect = get_dialect(schema_resolver.platform)
     else:
         dialect = get_dialect(default_dialect)
 
-    if is_dialect_instance(dialect, "snowflake"):
-        # in snowflake, table identifiers must be uppercased to match sqlglot's behavior.
-        if default_db:
-            default_db = default_db.upper()
-        if default_schema:
-            default_schema = default_schema.upper()
+    default_db = _normalize_db_or_schema(default_db, dialect)
+    default_schema = _normalize_db_or_schema(default_schema, dialect)
+
     if is_dialect_instance(dialect, "redshift") and not default_schema:
         # On Redshift, there's no "USE SCHEMA <schema>" command. The default schema
         # is public, and "current schema" is the one at the front of the search path.
@@ -886,12 +945,23 @@ def _sqlglot_lineage_inner(
     logger.debug("Parsing lineage from sql statement: %s", sql)
     statement = parse_statement(sql, dialect=dialect)
 
+    if isinstance(statement, sqlglot.exp.Command):
+        # For unsupported syntax, sqlglot will usually fallback to parsing as a Command.
+        # This is effectively a parsing error, and we won't get any lineage from it.
+        # See https://github.com/tobymao/sqlglot/commit/3a13fdf4e597a2f0a3f9fc126a129183fe98262f
+        # and https://github.com/tobymao/sqlglot/pull/2874
+        raise UnsupportedStatementTypeError(
+            f"Got unsupported syntax for statement: {sql}"
+        )
+
     original_statement, statement = statement, statement.copy()
     # logger.debug(
     #     "Formatted sql statement: %s",
     #     original_statement.sql(pretty=True, dialect=dialect),
     # )
 
+    statement = _simplify_select_into(statement)
+
     # Make sure the tables are resolved with the default db / schema.
     # This only works for Unionable statements. For other types of statements,
     # we have to do it manually afterwards, but that's slightly lower accuracy
@@ -905,6 +975,7 @@ def _sqlglot_lineage_inner(
         # At this stage we only want to qualify the table names. The columns will be dealt with later.
         qualify_columns=False,
         validate_qualify_columns=False,
+        allow_partial_qualification=True,
         # Only insert quotes where necessary.
         identify=False,
     )
@@ -953,6 +1024,14 @@ def _sqlglot_lineage_inner(
     logger.debug(
         f"Resolved {total_schemas_resolved} of {total_tables_discovered} table schemas"
     )
+    if SQL_PARSER_TRACE:
+        for qualified_table, schema_info in table_name_schema_mapping.items():
+            logger.debug(
+                "Table name %s resolved to %s with schema %s",
+                qualified_table,
+                table_name_urn_mapping[qualified_table],
+                schema_info,
+            )
 
     column_lineage: Optional[List[_ColumnLineageInfo]] = None
     try:
@@ -1112,6 +1191,45 @@ def sqlglot_lineage(
         )
 
 
+@functools.lru_cache(maxsize=128)
+def create_and_cache_schema_resolver(
+    platform: str,
+    env: str,
+    graph: Optional[DataHubGraph] = None,
+    platform_instance: Optional[str] = None,
+    schema_aware: bool = True,
+) -> SchemaResolver:
+    return create_schema_resolver(
+        platform=platform,
+        env=env,
+        graph=graph,
+        platform_instance=platform_instance,
+        schema_aware=schema_aware,
+    )
+
+
+def create_schema_resolver(
+    platform: str,
+    env: str,
+    graph: Optional[DataHubGraph] = None,
+    platform_instance: Optional[str] = None,
+    schema_aware: bool = True,
+) -> SchemaResolver:
+    if graph and schema_aware:
+        return graph._make_schema_resolver(
+            platform=platform,
+            platform_instance=platform_instance,
+            env=env,
+        )
+
+    return SchemaResolver(
+        platform=platform,
+        platform_instance=platform_instance,
+        env=env,
+        graph=None,
+    )
+
+
 def create_lineage_sql_parsed_result(
     query: str,
     default_db: Optional[str],
@@ -1122,21 +1240,17 @@ def create_lineage_sql_parsed_result(
     graph: Optional[DataHubGraph] = None,
     schema_aware: bool = True,
 ) -> SqlParsingResult:
+    schema_resolver = create_schema_resolver(
+        platform=platform,
+        platform_instance=platform_instance,
+        env=env,
+        schema_aware=schema_aware,
+        graph=graph,
+    )
+
+    needs_close: bool = True
     if graph and schema_aware:
         needs_close = False
-        schema_resolver = graph._make_schema_resolver(
-            platform=platform,
-            platform_instance=platform_instance,
-            env=env,
-        )
-    else:
-        needs_close = True
-        schema_resolver = SchemaResolver(
-            platform=platform,
-            platform_instance=platform_instance,
-            env=env,
-            graph=None,
-        )
 
     try:
         return sqlglot_lineage(
@@ -1174,13 +1288,19 @@ def infer_output_schema(result: SqlParsingResult) -> Optional[List[SchemaFieldCl
 def view_definition_lineage_helper(
     result: SqlParsingResult, view_urn: str
 ) -> SqlParsingResult:
-    if result.query_type is QueryType.SELECT:
+    if result.query_type is QueryType.SELECT or (
+        result.out_tables and result.out_tables != [view_urn]
+    ):
         # Some platforms (e.g. postgres) store only <select statement> from view definition
         # `create view V as <select statement>` . For such view definitions, `result.out_tables` and
         # `result.column_lineage[].downstream` are empty in `sqlglot_lineage` response, whereas upstream
         # details and downstream column details are extracted correctly.
         # Here, we inject view V's urn in `result.out_tables` and `result.column_lineage[].downstream`
         # to get complete lineage result.
+
+        # Some platforms(e.g. mssql) may have slightly different view name in view definition than
+        # actual view name used elsewhere. Therefore we overwrite downstream table for such cases as well.
+
         result.out_tables = [view_urn]
         if result.column_lineage:
             for col_result in result.column_lineage:
diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py
index 71245353101f6..57a5cc3c9a657 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py
@@ -1,3 +1,5 @@
+from datahub.sql_parsing._sqlglot_patch import SQLGLOT_PATCHED
+
 import functools
 import hashlib
 import logging
@@ -8,6 +10,8 @@
 import sqlglot.errors
 import sqlglot.optimizer.eliminate_ctes
 
+assert SQLGLOT_PATCHED
+
 logger = logging.getLogger(__name__)
 DialectOrStr = Union[sqlglot.Dialect, str]
 SQL_PARSE_CACHE_SIZE = 1000
@@ -35,6 +39,9 @@ def _get_dialect_str(platform: str) -> str:
         # let the fuzzy resolution logic handle it.
         # MariaDB is a fork of MySQL, so we reuse the same dialect.
         return "mysql, normalization_strategy = lowercase"
+    # Dremio is based upon drill. Not 100% compatibility
+    elif platform == "dremio":
+        return "drill"
     else:
         return platform
 
@@ -54,7 +61,7 @@ def is_dialect_instance(
     else:
         platforms = list(platforms)
 
-    dialects = [sqlglot.Dialect.get_or_raise(platform) for platform in platforms]
+    dialects = [get_dialect(platform) for platform in platforms]
 
     if any(isinstance(dialect, dialect_class.__class__) for dialect_class in dialects):
         return True
@@ -114,7 +121,7 @@ def _expression_to_string(
     # Remove /* */ comments.
     re.compile(r"/\*.*?\*/", re.DOTALL): "",
     # Remove -- comments.
-    re.compile(r"--.*$"): "",
+    re.compile(r"--.*$", re.MULTILINE): "",
     # Replace all runs of whitespace with a single space.
     re.compile(r"\s+"): " ",
     # Remove leading and trailing whitespace and trailing semicolons.
@@ -124,10 +131,16 @@ def _expression_to_string(
     # Replace anything that looks like a string with a placeholder.
     re.compile(r"'[^']*'"): "?",
     # Replace sequences of IN/VALUES with a single placeholder.
-    re.compile(r"\b(IN|VALUES)\s*\(\?(?:, \?)*\)", re.IGNORECASE): r"\1 (?)",
+    # The r" ?" makes it more robust to uneven spacing.
+    re.compile(r"\b(IN|VALUES)\s*\( ?\?(?:, ?\?)* ?\)", re.IGNORECASE): r"\1 (?)",
     # Normalize parenthesis spacing.
     re.compile(r"\( "): "(",
     re.compile(r" \)"): ")",
+    # Fix up spaces before commas in column lists.
+    # e.g. "col1 , col2" -> "col1, col2"
+    # e.g. "col1,col2" -> "col1, col2"
+    re.compile(r"\b ,"): ",",
+    re.compile(r"\b,\b"): ", ",
 }
 _TABLE_NAME_NORMALIZATION_RULES = {
     # Replace UUID-like strings with a placeholder (both - and _ variants).
diff --git a/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py b/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py
index cdd35c23e3088..5af9d9d4f0fff 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py
@@ -1,3 +1,4 @@
+import contextlib
 import json
 import logging
 from dataclasses import dataclass, field
@@ -5,8 +6,15 @@
 
 from typing_extensions import Protocol
 
+from datahub.api.entities.platformresource.platform_resource import (
+    ElasticPlatformResourceQuery,
+    PlatformResource,
+    PlatformResourceSearchFields,
+)
 from datahub.ingestion.api.report import Report
+from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.urns import CorpGroupUrn, CorpUserUrn
+from datahub.utilities.search_utils import LogicalOperator
 from datahub.utilities.stats_collections import int_top_k_dict
 
 UrnStr = str
@@ -31,6 +39,7 @@ def _get_last_line(query: str) -> str:
 @dataclass
 class ToolMetaExtractorReport(Report):
     num_queries_meta_extracted: Dict[str, int] = field(default_factory=int_top_k_dict)
+    failures: List[str] = field(default_factory=list)
 
 
 class ToolMetaExtractor:
@@ -42,14 +51,81 @@ class ToolMetaExtractor:
     by warehouse query logs.
     """
 
-    def __init__(self) -> None:
-        self.report = ToolMetaExtractorReport()
+    def __init__(
+        self,
+        report: ToolMetaExtractorReport,
+        looker_user_mapping: Optional[Dict[str, str]] = None,
+    ) -> None:
+        self.report = report
         self.known_tool_extractors: List[Tuple[str, Callable[[QueryLog], bool]]] = [
             (
                 "mode",
                 self._extract_mode_query,
-            )
+            ),
+            (
+                "looker",
+                self._extract_looker_query,
+            ),
         ]
+        # maps user id (as string) to email address
+        self.looker_user_mapping = looker_user_mapping
+
+    @classmethod
+    def create(
+        cls,
+        graph: Optional[DataHubGraph] = None,
+    ) -> "ToolMetaExtractor":
+        report = ToolMetaExtractorReport()
+        looker_user_mapping = None
+        if graph:
+            try:
+                looker_user_mapping = cls.extract_looker_user_mapping_from_graph(
+                    graph, report
+                )
+            except Exception as e:
+                report.failures.append(
+                    f"Unexpected error during Looker user metadata extraction: {str(e)}"
+                )
+
+        return cls(report, looker_user_mapping)
+
+    @classmethod
+    def extract_looker_user_mapping_from_graph(
+        cls, graph: DataHubGraph, report: ToolMetaExtractorReport
+    ) -> Optional[Dict[str, str]]:
+        looker_user_mapping = None
+        query = (
+            ElasticPlatformResourceQuery.create_from()
+            .group(LogicalOperator.AND)
+            .add_field_match(PlatformResourceSearchFields.PLATFORM, "looker")
+            .add_field_match(
+                PlatformResourceSearchFields.RESOURCE_TYPE,
+                "USER_ID_MAPPING",
+            )
+            .end()
+        )
+        platform_resources = list(
+            PlatformResource.search_by_filters(query=query, graph_client=graph)
+        )
+
+        if len(platform_resources) > 1:
+            report.failures.append(
+                "Looker user metadata extraction failed. Found more than one looker user id mappings."
+            )
+        else:
+            platform_resource = platform_resources[0]
+
+            if (
+                platform_resource
+                and platform_resource.resource_info
+                and platform_resource.resource_info.value
+            ):
+                with contextlib.suppress(ValueError, AssertionError):
+                    value = platform_resource.resource_info.value.as_raw_json()
+                    if value:
+                        looker_user_mapping = value
+
+        return looker_user_mapping
 
     def _extract_mode_query(self, entry: QueryLog) -> bool:
         """
@@ -78,15 +154,49 @@ def _extract_mode_query(self, entry: QueryLog) -> bool:
 
         return True
 
-    def extract_bi_metadata(self, entry: QueryLog) -> bool:
+    def _extract_looker_query(self, entry: QueryLog) -> bool:
+        """
+        Returns:
+            bool: whether QueryLog entry is that of looker and looker user info
+            is extracted into entry.
+        """
+        if not self.looker_user_mapping:
+            return False
+
+        last_line = _get_last_line(entry.query_text)
 
+        if not (last_line.startswith("--") and "Looker Query Context" in last_line):
+            return False
+
+        start_quote_idx = last_line.index("'")
+        end_quote_idx = last_line.rindex("'")
+        if start_quote_idx == -1 or end_quote_idx == -1:
+            return False
+
+        looker_json_raw = last_line[start_quote_idx + 1 : end_quote_idx]
+        looker_json = json.loads(looker_json_raw)
+
+        user_id = str(looker_json["user_id"])
+        email = self.looker_user_mapping.get(user_id)
+        if not email:
+            return False
+
+        original_user = entry.user
+
+        entry.user = email_to_user_urn(email)
+        entry.extra_info = entry.extra_info or {}
+        entry.extra_info["user_via"] = original_user
+
+        return True
+
+    def extract_bi_metadata(self, entry: QueryLog) -> bool:
         for tool, meta_extractor in self.known_tool_extractors:
             try:
                 if meta_extractor(entry):
                     self.report.num_queries_meta_extracted[tool] += 1
                     return True
-            except Exception:
-                logger.debug("Tool metadata extraction failed with error : {e}")
+            except Exception as e:
+                logger.debug(f"Tool metadata extraction failed with error : {e}")
         return False
 
 
diff --git a/metadata-ingestion/src/datahub/telemetry/telemetry.py b/metadata-ingestion/src/datahub/telemetry/telemetry.py
index 4faf04ee2d2c7..22b2cb6a101af 100644
--- a/metadata-ingestion/src/datahub/telemetry/telemetry.py
+++ b/metadata-ingestion/src/datahub/telemetry/telemetry.py
@@ -7,7 +7,7 @@
 import uuid
 from functools import wraps
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, TypeVar
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar
 
 from mixpanel import Consumer, Mixpanel
 from typing_extensions import ParamSpec
@@ -16,10 +16,12 @@
 from datahub.cli.config_utils import DATAHUB_ROOT_FOLDER
 from datahub.cli.env_utils import get_boolean_env_variable
 from datahub.configuration.common import ExceptionWithProps
-from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import _custom_package_path
 from datahub.utilities.perf_timer import PerfTimer
 
+if TYPE_CHECKING:
+    from datahub.ingestion.graph.client import DataHubGraph
+
 logger = logging.getLogger(__name__)
 
 DATAHUB_FOLDER = Path(DATAHUB_ROOT_FOLDER)
@@ -117,7 +119,11 @@ class Telemetry:
     tracking_init: bool = False
     sentry_enabled: bool = False
 
+    context_properties: Dict[str, Any] = {}
+
     def __init__(self):
+        self.context_properties = {}
+
         if SENTRY_DSN:
             self.sentry_enabled = True
             try:
@@ -157,6 +163,9 @@ def __init__(self):
             except Exception as e:
                 logger.debug(f"Error connecting to mixpanel: {e}")
 
+        # Initialize the default properties for all events.
+        self.set_context()
+
     def update_config(self) -> bool:
         """
         Update the config file with the current client ID and enabled status.
@@ -238,18 +247,22 @@ def load_config(self) -> bool:
 
         return False
 
-    def update_capture_exception_context(
+    def set_context(
         self,
-        server: Optional[DataHubGraph] = None,
+        server: Optional["DataHubGraph"] = None,
         properties: Optional[Dict[str, Any]] = None,
     ) -> None:
+        self.context_properties = {
+            **self._server_props(server),
+            **(properties or {}),
+        }
+
         if self.sentry_enabled:
             from sentry_sdk import set_tag
 
             properties = {
                 **_default_telemetry_properties(),
-                **self._server_props(server),
-                **(properties or {}),
+                **self.context_properties,
             }
 
             for key in properties:
@@ -297,7 +310,6 @@ def ping(
         self,
         event_name: str,
         properties: Optional[Dict[str, Any]] = None,
-        server: Optional[DataHubGraph] = None,
     ) -> None:
         """
         Send a single telemetry event.
@@ -323,14 +335,15 @@ def ping(
 
             properties = {
                 **_default_telemetry_properties(),
-                **self._server_props(server),
+                **self.context_properties,
                 **properties,
             }
             self.mp.track(self.client_id, event_name, properties)
         except Exception as e:
             logger.debug(f"Error reporting telemetry: {e}")
 
-    def _server_props(self, server: Optional[DataHubGraph]) -> Dict[str, str]:
+    @classmethod
+    def _server_props(cls, server: Optional["DataHubGraph"]) -> Dict[str, str]:
         if not server:
             return {
                 "server_type": "n/a",
@@ -435,6 +448,7 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:
                             **call_props,
                             "status": "error",
                             **_error_props(e),
+                            "code": e.code,
                         },
                     )
                 telemetry_instance.capture_exception(e)
diff --git a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
index 72b5f6c5e26e4..13be45ec1be28 100644
--- a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
+++ b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
@@ -6,12 +6,8 @@
 import deepdiff
 
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier
-from datahub.sql_parsing.schema_resolver import SchemaResolver
-from datahub.sql_parsing.sqlglot_lineage import (
-    SchemaInfo,
-    SqlParsingResult,
-    sqlglot_lineage,
-)
+from datahub.sql_parsing.schema_resolver import SchemaInfo, SchemaResolver
+from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult, sqlglot_lineage
 
 logger = logging.getLogger(__name__)
 
diff --git a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py
index 61b222f8d2dd5..9dbadd4804997 100644
--- a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py
+++ b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py
@@ -27,6 +27,8 @@
     r"root\[\d+\]\['aspect'\]\['json'\]\['lastUpdatedTimestamp'\]",
     r"root\[\d+\]\['aspect'\]\['json'\]\['created'\]",
     r"root\[\d+\]\['aspect'\]\['json'\]\['lastModified'\]",
+    r"root\[\d+\].*?\['systemMetadata'\]\['runId'\]",
+    r"root\[\d+\].*?\['systemMetadata'\]\['lastRunId'\]",
 ]
 
 
@@ -82,13 +84,17 @@ def assert_metadata_files_equal(
                         json_path = f"root[{i}]['aspect']['json'][{j}]['value']"
                         ignore_paths = (*ignore_paths, re.escape(json_path))
 
+    ignore_paths = (*ignore_paths, *default_exclude_paths)
+
     diff = diff_metadata_json(output, golden, ignore_paths, ignore_order=ignore_order)
     if diff and update_golden:
         if isinstance(diff, MCPDiff) and diff.is_delta_valid:
+            logger.info(f"Applying delta to golden file {golden_path}")
             diff.apply_delta(golden)
             write_metadata_file(pathlib.Path(golden_path), golden)
         else:
             # Fallback: just overwrite the golden file
+            logger.info(f"Overwriting golden file {golden_path}")
             shutil.copyfile(str(output_path), str(golden_path))
         return
 
@@ -111,7 +117,7 @@ def diff_metadata_json(
     ignore_paths: Sequence[str] = (),
     ignore_order: bool = True,
 ) -> Union[DeepDiff, MCPDiff]:
-    ignore_paths = (*ignore_paths, *default_exclude_paths, r"root\[\d+].delta_info")
+    ignore_paths = [*ignore_paths, *default_exclude_paths, r"root\[\d+].delta_info"]
     try:
         if ignore_order:
             golden_map = get_aspects_by_urn(golden)
diff --git a/metadata-ingestion/src/datahub/testing/doctest.py b/metadata-ingestion/src/datahub/testing/doctest.py
new file mode 100644
index 0000000000000..b89df5c65c7e1
--- /dev/null
+++ b/metadata-ingestion/src/datahub/testing/doctest.py
@@ -0,0 +1,12 @@
+import doctest
+from types import ModuleType
+
+
+def assert_doctest(module: ModuleType) -> None:
+    result = doctest.testmod(
+        module,
+        raise_on_error=True,
+        verbose=True,
+    )
+    if result.attempted == 0:
+        raise ValueError(f"No doctests found in {module.__name__}")
diff --git a/metadata-ingestion/src/datahub/testing/mcp_diff.py b/metadata-ingestion/src/datahub/testing/mcp_diff.py
index 95b8e83c7a64a..5e669a718e9ad 100644
--- a/metadata-ingestion/src/datahub/testing/mcp_diff.py
+++ b/metadata-ingestion/src/datahub/testing/mcp_diff.py
@@ -206,7 +206,7 @@ def apply_delta(self, golden: List[Dict[str, Any]]) -> None:
         """
         aspect_diffs = [v for d in self.aspect_changes.values() for v in d.values()]
         for aspect_diff in aspect_diffs:
-            for (_, old, new) in aspect_diff.aspects_changed.keys():
+            for _, old, new in aspect_diff.aspects_changed.keys():
                 golden[old.delta_info.idx] = new.delta_info.original
 
         indices_to_remove = set()
diff --git a/metadata-ingestion/src/datahub/upgrade/upgrade.py b/metadata-ingestion/src/datahub/upgrade/upgrade.py
index d940dfd78a82e..fb14514588e5f 100644
--- a/metadata-ingestion/src/datahub/upgrade/upgrade.py
+++ b/metadata-ingestion/src/datahub/upgrade/upgrade.py
@@ -1,19 +1,19 @@
 import asyncio
 import contextlib
 import logging
-import sys
 from datetime import datetime, timedelta, timezone
 from functools import wraps
 from typing import Any, Callable, Optional, Tuple, TypeVar
 
+import click
 import humanfriendly
 from packaging.version import Version
 from pydantic import BaseModel
-from termcolor import colored
 
 from datahub import __version__
 from datahub.cli.config_utils import load_client_config
 from datahub.ingestion.graph.client import DataHubGraph
+from datahub.utilities.perf_timer import PerfTimer
 
 log = logging.getLogger(__name__)
 
@@ -114,7 +114,7 @@ async def get_server_config(gms_url: str, token: Optional[str]) -> dict:
 
     async with aiohttp.ClientSession() as session:
         config_endpoint = f"{gms_url}/config"
-        async with session.get(config_endpoint) as dh_response:
+        async with session.get(config_endpoint, headers=headers) as dh_response:
             dh_response_json = await dh_response.json()
             return dh_response_json
 
@@ -168,7 +168,28 @@ async def get_server_version_stats(
     return (server_type, server_version, current_server_release_date)
 
 
-async def retrieve_version_stats(
+def retrieve_version_stats(
+    timeout: float, graph: Optional[DataHubGraph] = None
+) -> Optional[DataHubVersionStats]:
+    version_stats: Optional[DataHubVersionStats] = None
+
+    async def _get_version_with_timeout() -> None:
+        # TODO: Once we're on Python 3.11+, replace with asyncio.timeout.
+        stats_future = _retrieve_version_stats(graph)
+
+        try:
+            nonlocal version_stats
+            version_stats = await asyncio.wait_for(stats_future, timeout=timeout)
+        except asyncio.TimeoutError:
+            log.debug("Timed out while fetching version stats")
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(_get_version_with_timeout())
+
+    return version_stats
+
+
+async def _retrieve_version_stats(
     server: Optional[DataHubGraph] = None,
 ) -> Optional[DataHubVersionStats]:
     try:
@@ -264,7 +285,7 @@ def is_client_server_compatible(client: VersionStats, server: VersionStats) -> i
         return server.version.micro - client.version.micro
 
 
-def maybe_print_upgrade_message(  # noqa: C901
+def _maybe_print_upgrade_message(  # noqa: C901
     version_stats: Optional[DataHubVersionStats],
 ) -> None:  # noqa: C901
     days_before_cli_stale = 7
@@ -277,8 +298,8 @@ def maybe_print_upgrade_message(  # noqa: C901
         if not version_stats:
             log.debug("No version stats found")
             return
-        else:
-            log.debug(f"Version stats found: {version_stats}")
+
+        log.debug(f"Version stats found: {version_stats}")
         current_release_date = version_stats.client.current.release_date
         latest_release_date = (
             version_stats.client.latest.release_date
@@ -325,78 +346,88 @@ def maybe_print_upgrade_message(  # noqa: C901
     if client_server_compat < 0:
         with contextlib.suppress(Exception):
             assert version_stats
-            print(
-                colored("❗Client-Server Incompatible❗", "yellow"),
-                colored(
+            click.echo(
+                click.style("❗Client-Server Incompatible❗", fg="yellow")
+                + " "
+                + click.style(
                     f"Your client version {version_stats.client.current.version} is newer than your server version {version_stats.server.current.version}. Downgrading the cli to {version_stats.server.current.version} is recommended.\n",
-                    "cyan",
-                ),
-                colored(
+                    fg="cyan",
+                )
+                + click.style(
                     f"➡️ Downgrade via `\"pip install 'acryl-datahub=={version_stats.server.current.version}'\"",
-                    "cyan",
-                ),
+                    fg="cyan",
+                )
             )
     elif client_server_compat > 0:
         with contextlib.suppress(Exception):
             assert version_stats
-            print(
-                colored("❗Client-Server Incompatible❗", "red"),
-                colored(
+            click.echo(
+                click.style("❗Client-Server Incompatible❗", fg="red")
+                + " "
+                + click.style(
                     f"Your client version {version_stats.client.current.version} is older than your server version {version_stats.server.current.version}. Upgrading the cli to {version_stats.server.current.version} is recommended.\n",
-                    "cyan",
-                ),
-                colored(
+                    fg="cyan",
+                )
+                + click.style(
                     f"➡️  Upgrade via \"pip install 'acryl-datahub=={version_stats.server.current.version}'\"",
-                    "cyan",
-                ),
+                    fg="cyan",
+                )
             )
     elif client_server_compat == 0 and encourage_cli_upgrade:
         with contextlib.suppress(Exception):
-            print(
-                colored("💡 Upgrade cli!", "yellow"),
-                colored(
+            click.echo(
+                click.style("💡 Upgrade cli!", fg="yellow")
+                + " "
+                + click.style(
                     f"You seem to be running an old version of datahub cli: {current_version} {get_days(current_release_date)}. Latest version is {latest_version} {get_days(latest_release_date)}.\nUpgrade via \"pip install -U 'acryl-datahub'\"",
-                    "cyan",
-                ),
+                    fg="cyan",
+                )
             )
     elif encourage_quickstart_upgrade:
         try:
             assert version_stats
-            print(
-                colored("💡 Upgrade available!", "yellow"),
-                colored(
+            click.echo(
+                click.style("💡 Upgrade available!", fg="yellow")
+                + " "
+                + click.style(
                     f'You seem to be running a slightly old quickstart image {get_days(version_stats.server.current.release_date)}. Run "datahub docker quickstart" to get the latest updates without losing any data!',
-                    "cyan",
+                    fg="cyan",
                 ),
-                file=sys.stderr,
+                err=True,
             )
         except Exception as e:
             log.debug(f"Failed to suggest quickstart upgrade due to {e}")
             pass
 
 
+def clip(val: float, min_val: float, max_val: float) -> float:
+    return max(min_val, min(val, max_val))
+
+
+def check_upgrade_post(
+    main_method_runtime: float,
+    graph: Optional[DataHubGraph] = None,
+) -> None:
+    # Guarantees: this method will not throw, and will not block for more than 3 seconds.
+
+    version_stats_timeout = clip(main_method_runtime / 10, 0.7, 3.0)
+    try:
+        version_stats = retrieve_version_stats(
+            timeout=version_stats_timeout, graph=graph
+        )
+        _maybe_print_upgrade_message(version_stats=version_stats)
+    except Exception as e:
+        log.debug(f"Failed to check for upgrades due to {e}")
+
+
 def check_upgrade(func: Callable[..., T]) -> Callable[..., T]:
     @wraps(func)
     def async_wrapper(*args: Any, **kwargs: Any) -> Any:
-        async def run_inner_func():
-            return func(*args, **kwargs)
-
-        async def run_func_check_upgrade():
-            version_stats_future = asyncio.ensure_future(retrieve_version_stats())
-            main_func_future = asyncio.ensure_future(run_inner_func())
-            ret = await main_func_future
-
-            # the main future has returned
-            # we check the other futures quickly
-            try:
-                version_stats = await asyncio.wait_for(version_stats_future, 0.5)
-                maybe_print_upgrade_message(version_stats=version_stats)
-            except Exception:
-                log.debug("timed out waiting for version stats to be computed")
+        with PerfTimer() as timer:
+            ret = func(*args, **kwargs)
 
-            return ret
+        check_upgrade_post(main_method_runtime=timer.elapsed_seconds())
 
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(run_func_check_upgrade())
+        return ret
 
     return async_wrapper
diff --git a/metadata-ingestion/src/datahub/utilities/cachetools_keys.py b/metadata-ingestion/src/datahub/utilities/cachetools_keys.py
new file mode 100644
index 0000000000000..e3c7d67c81cd3
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/cachetools_keys.py
@@ -0,0 +1,8 @@
+from typing import Any
+
+import cachetools.keys
+
+
+def self_methodkey(self: Any, *args: Any, **kwargs: Any) -> Any:
+    # Keeps the id of self around
+    return cachetools.keys.hashkey(id(self), *args, **kwargs)
diff --git a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py
index b67f9c1a9ffa1..b8c27666d7f53 100644
--- a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py
+++ b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py
@@ -1,6 +1,7 @@
 import collections
 import gzip
 import logging
+import os
 import pathlib
 import pickle
 import shutil
@@ -33,6 +34,14 @@
 
 logger: logging.Logger = logging.getLogger(__name__)
 
+OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR = (
+    os.environ.get("OVERRIDE_SQLITE_VERSION_REQ") or ""
+)
+OVERRIDE_SQLITE_VERSION_REQUIREMENT = (
+    OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR
+    and OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR.lower() != "false"
+)
+
 _DEFAULT_FILE_NAME = "sqlite.db"
 _DEFAULT_TABLE_NAME = "data"
 
@@ -212,6 +221,7 @@ class FileBackedDict(MutableMapping[str, _VT], Closeable, Generic[_VT]):
     _active_object_cache: OrderedDict[str, Tuple[_VT, bool]] = field(
         init=False, repr=False
     )
+    _use_sqlite_on_conflict: bool = field(repr=False, default=True)
 
     def __post_init__(self) -> None:
         assert (
@@ -228,6 +238,15 @@ def __post_init__(self) -> None:
         else:
             self._conn = ConnectionWrapper()
 
+        if sqlite3.sqlite_version_info < (3, 24, 0):
+            # We use the ON CONFLICT clause to implement UPSERTs with sqlite.
+            # This was added in 3.24.0 from 2018-06-04.
+            # See https://www.sqlite.org/lang_conflict.html
+            if OVERRIDE_SQLITE_VERSION_REQUIREMENT:
+                self.use_sqlite_on_conflict = False
+            else:
+                raise RuntimeError("SQLite version 3.24.0 or later is required")
+
         # We keep a small cache in memory to avoid having to serialize/deserialize
         # data from the database too often. We use an OrderedDict to build
         # a poor-man's LRU cache.
@@ -289,7 +308,7 @@ def _prune_cache(self, num_items_to_prune: int) -> None:
                     values.append(column_serializer(value))
                 items_to_write.append(tuple(values))
 
-        if items_to_write:
+        if items_to_write and self._use_sqlite_on_conflict:
             # Tricky: By using a INSERT INTO ... ON CONFLICT (key) structure, we can
             # ensure that the rowid remains the same if a value is updated but is
             # autoincremented when rows are inserted.
@@ -306,6 +325,26 @@ def _prune_cache(self, num_items_to_prune: int) -> None:
                 """,
                 items_to_write,
             )
+        else:
+            for item in items_to_write:
+                try:
+                    self._conn.execute(
+                        f"""INSERT INTO {self.tablename} (
+                            key,
+                            value
+                            {''.join(f', {column_name}' for column_name in self.extra_columns.keys())}
+                        )
+                        VALUES ({', '.join(['?'] *(2 + len(self.extra_columns)))})""",
+                        item,
+                    )
+                except sqlite3.IntegrityError:
+                    self._conn.execute(
+                        f"""UPDATE {self.tablename} SET
+                            value = ?
+                            {''.join(f', {column_name} = ?' for column_name in self.extra_columns.keys())}
+                        WHERE key = ?""",
+                        (*item[1:], item[0]),
+                    )
 
     def flush(self) -> None:
         self._prune_cache(len(self._active_object_cache))
diff --git a/metadata-ingestion/src/datahub/utilities/is_pytest.py b/metadata-ingestion/src/datahub/utilities/is_pytest.py
new file mode 100644
index 0000000000000..572b4bf535622
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/is_pytest.py
@@ -0,0 +1,6 @@
+import os
+import sys
+
+
+def is_pytest_running() -> bool:
+    return "pytest" in sys.modules and os.environ.get("DATAHUB_TEST_MODE") == "1"
diff --git a/metadata-ingestion/src/datahub/utilities/mapping.py b/metadata-ingestion/src/datahub/utilities/mapping.py
index 4ea42d568da63..17023c7b388e7 100644
--- a/metadata-ingestion/src/datahub/utilities/mapping.py
+++ b/metadata-ingestion/src/datahub/utilities/mapping.py
@@ -43,7 +43,6 @@ def _make_owner_category_list(
     owner_category_urn: Optional[str],
     owner_ids: List[str],
 ) -> List[Dict]:
-
     return [
         {
             "urn": mce_builder.make_owner_urn(owner_id, owner_type),
@@ -285,7 +284,6 @@ def convert_to_aspects(self, operation_map: Dict[str, list]) -> Dict[str, Any]:
             aspect_map[Constants.ADD_TAG_OPERATION] = tag_aspect
 
         if Constants.ADD_OWNER_OPERATION in operation_map:
-
             owner_aspect = OwnershipClass(
                 owners=[
                     OwnerClass(
diff --git a/metadata-ingestion/src/datahub/utilities/openapi_utils.py b/metadata-ingestion/src/datahub/utilities/openapi_utils.py
new file mode 100644
index 0000000000000..e704ff7f84cbb
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/openapi_utils.py
@@ -0,0 +1,69 @@
+import logging
+from typing import Iterable, Union
+
+import datahub.metadata.schema_classes as models
+from datahub.ingestion.graph.client import DataHubGraph
+from datahub.utilities.search_utils import (
+    ElasticDocumentQuery,
+    ElasticsearchQueryBuilder,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAPIGraphClient:
+    """
+    An experimental client for the DataHubGraph that uses the OpenAPI endpoints
+    to query entities and aspects.
+    Does not support all features of the DataHubGraph.
+    API is subject to change.
+
+    DO NOT USE THIS UNLESS YOU KNOW WHAT YOU ARE DOING.
+    """
+
+    ENTITY_KEY_ASPECT_MAP = {
+        aspect_type.ASPECT_INFO.get("keyForEntity"): name
+        for name, aspect_type in models.ASPECT_NAME_MAP.items()
+        if aspect_type.ASPECT_INFO.get("keyForEntity")
+    }
+
+    def __init__(self, graph: DataHubGraph):
+        self.graph = graph
+        self.openapi_base = graph._gms_server.rstrip("/") + "/openapi/v3"
+
+    def scroll_urns_by_filter(
+        self,
+        entity_type: str,
+        query: Union[ElasticDocumentQuery, ElasticsearchQueryBuilder],
+    ) -> Iterable[str]:
+        """
+        Scroll through all urns that match the given filters.
+
+        """
+
+        key_aspect = self.ENTITY_KEY_ASPECT_MAP.get(entity_type)
+        assert key_aspect, f"No key aspect found for entity type {entity_type}"
+
+        count = 1000
+        string_query = query.build()
+        scroll_id = None
+        logger.debug(f"Scrolling with query: {string_query}")
+        while True:
+            response = self.graph._get_generic(
+                self.openapi_base + f"/entity/{entity_type.lower()}",
+                params={
+                    "systemMetadata": "false",
+                    "includeSoftDelete": "false",
+                    "skipCache": "false",
+                    "aspects": [key_aspect],
+                    "scrollId": scroll_id,
+                    "count": count,
+                    "query": string_query,
+                },
+            )
+            entities = response.get("entities", [])
+            scroll_id = response.get("scrollId")
+            for entity in entities:
+                yield entity["urn"]
+            if not scroll_id:
+                break
diff --git a/metadata-ingestion/src/datahub/utilities/partition_executor.py b/metadata-ingestion/src/datahub/utilities/partition_executor.py
index 92413eeb674f5..542889f2f90e2 100644
--- a/metadata-ingestion/src/datahub/utilities/partition_executor.py
+++ b/metadata-ingestion/src/datahub/utilities/partition_executor.py
@@ -53,7 +53,10 @@ def __init__(self, max_workers: int, max_pending: int) -> None:
         self.max_workers = max_workers
         self.max_pending = max_pending
 
-        self._executor = ThreadPoolExecutor(max_workers=max_workers)
+        self._executor = ThreadPoolExecutor(
+            max_workers=max_workers,
+            thread_name_prefix=self.__class__.__name__,
+        )
 
         # Each pending or executing request will acquire a permit from this semaphore.
         self._semaphore = BoundedSemaphore(max_pending + max_workers)
@@ -234,6 +237,11 @@ def __init__(
         process_batch: Callable[[List], None],
         max_per_batch: int = 100,
         min_process_interval: timedelta = _DEFAULT_BATCHER_MIN_PROCESS_INTERVAL,
+        # Why 3 seconds? It's somewhat arbitrary.
+        # We don't want it to be too high, since then liveness suffers,
+        # particularly during a dirty shutdown. If it's too low, then we'll
+        # waste CPU cycles rechecking the timer, only to call get again.
+        read_from_pending_interval: timedelta = timedelta(seconds=3),
     ) -> None:
         """Similar to PartitionExecutor, but with batching.
 
@@ -259,13 +267,21 @@ def __init__(
         self.max_per_batch = max_per_batch
         self.process_batch = process_batch
         self.min_process_interval = min_process_interval
-        assert self.max_workers > 1
-
-        # We add one here to account for the clearinghouse worker thread.
-        self._executor = ThreadPoolExecutor(max_workers=max_workers + 1)
+        self.read_from_pending_interval = read_from_pending_interval
+        assert self.max_workers >= 1
+
+        self._state_lock = threading.Lock()
+        self._executor = ThreadPoolExecutor(
+            # We add one here to account for the clearinghouse worker thread.
+            max_workers=max_workers + 1,
+            thread_name_prefix=self.__class__.__name__,
+        )
         self._clearinghouse_started = False
 
+        # pending_count includes the length of the pending list, plus the
+        # number of items sitting in the clearinghouse's internal queue.
         self._pending_count = BoundedSemaphore(max_pending)
+
         self._pending: "queue.Queue[Optional[_BatchPartitionWorkItem]]" = queue.Queue(
             maxsize=max_pending
         )
@@ -295,6 +311,9 @@ def _handle_batch_completion(
             batch: List[_BatchPartitionWorkItem], future: Future
         ) -> None:
             with clearinghouse_state_lock:
+                nonlocal workers_available
+                workers_available += 1
+
                 for item in batch:
                     keys_no_longer_in_flight.add(item.key)
                     self._pending_count.release()
@@ -304,7 +323,7 @@ def _handle_batch_completion(
                 if item.done_callback:
                     item.done_callback(future)
 
-        def _find_ready_items() -> List[_BatchPartitionWorkItem]:
+        def _find_ready_items(max_to_add: int) -> List[_BatchPartitionWorkItem]:
             with clearinghouse_state_lock:
                 # First, update the keys in flight.
                 for key in keys_no_longer_in_flight:
@@ -317,10 +336,7 @@ def _find_ready_items() -> List[_BatchPartitionWorkItem]:
 
                 ready: List[_BatchPartitionWorkItem] = []
                 for item in pending:
-                    if (
-                        len(ready) < self.max_per_batch
-                        and item.key not in keys_in_flight
-                    ):
+                    if len(ready) < max_to_add and item.key not in keys_in_flight:
                         ready.append(item)
                     else:
                         pending_key_completion.append(item)
@@ -328,7 +344,7 @@ def _find_ready_items() -> List[_BatchPartitionWorkItem]:
                 return ready
 
         def _build_batch() -> List[_BatchPartitionWorkItem]:
-            next_batch = _find_ready_items()
+            next_batch = _find_ready_items(self.max_per_batch)
 
             while (
                 not self._queue_empty_for_shutdown
@@ -350,12 +366,8 @@ def _build_batch() -> List[_BatchPartitionWorkItem]:
                     if not blocking:
                         next_item = self._pending.get_nowait()
                     else:
-                        # Why 3 seconds? It's somewhat arbitrary.
-                        # We don't want it to be too high, since then liveness suffers,
-                        # particularly during a dirty shutdown. If it's too low, then we'll
-                        # waste CPU cycles rechecking the timer, only to call get again.
                         next_item = self._pending.get(
-                            timeout=3,  # seconds
+                            timeout=self.read_from_pending_interval.total_seconds(),
                         )
 
                     if next_item is None:  # None is the shutdown signal
@@ -368,7 +380,11 @@ def _build_batch() -> List[_BatchPartitionWorkItem]:
                         else:
                             next_batch.append(next_item)
                 except queue.Empty:
-                    if not blocking:
+                    if blocking:
+                        next_batch.extend(
+                            _find_ready_items(self.max_per_batch - len(next_batch))
+                        )
+                    else:
                         break
 
             return next_batch
@@ -440,10 +456,11 @@ def _ensure_clearinghouse_started(self) -> None:
                 f"{self.__class__.__name__} is shutting down; cannot submit new work items."
             )
 
-        # Lazily start the clearinghouse worker.
-        if not self._clearinghouse_started:
-            self._clearinghouse_started = True
-            self._executor.submit(self._clearinghouse_worker)
+        with self._state_lock:
+            # Lazily start the clearinghouse worker.
+            if not self._clearinghouse_started:
+                self._clearinghouse_started = True
+                self._executor.submit(self._clearinghouse_worker)
 
     def submit(
         self,
diff --git a/metadata-ingestion/src/datahub/utilities/perf_timer.py b/metadata-ingestion/src/datahub/utilities/perf_timer.py
index 18384420bfefb..9488683d6d8ca 100644
--- a/metadata-ingestion/src/datahub/utilities/perf_timer.py
+++ b/metadata-ingestion/src/datahub/utilities/perf_timer.py
@@ -9,7 +9,6 @@
 class PerfTimer(AbstractContextManager):
     """
     A context manager that gives easy access to elapsed time for performance measurement.
-
     """
 
     def __init__(self) -> None:
diff --git a/metadata-ingestion/src/datahub/utilities/progress_timer.py b/metadata-ingestion/src/datahub/utilities/progress_timer.py
new file mode 100644
index 0000000000000..eac62cddb55f2
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/progress_timer.py
@@ -0,0 +1,34 @@
+from datetime import datetime, timedelta, timezone
+
+
+class ProgressTimer:
+    def __init__(self, report_every: timedelta, report_0: bool = False):
+        """A helper for reporting progress at a given time interval.
+
+        Should be used for long-running processes that iterate over a large number of items,
+        but each iteration is fast.
+
+        Args:
+            report_every: The time interval between progress reports.
+            report_0: Whether to report progress on the first iteration.
+        """
+
+        self._report_every = report_every
+
+        if report_0:
+            # Use the earliest possible time to force reporting on the first iteration.
+            self._last_report_time = datetime.min.replace(tzinfo=timezone.utc)
+        else:
+            self._last_report_time = self._now()
+
+    def _now(self) -> datetime:
+        return datetime.now(timezone.utc)
+
+    def should_report(self) -> bool:
+        current_time = self._now()
+
+        should_report = (self._last_report_time + self._report_every) <= current_time
+        if should_report:
+            self._last_report_time = current_time
+
+        return should_report
diff --git a/metadata-ingestion/src/datahub/utilities/search_utils.py b/metadata-ingestion/src/datahub/utilities/search_utils.py
new file mode 100644
index 0000000000000..0bd88addd8660
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/search_utils.py
@@ -0,0 +1,285 @@
+import logging
+import re
+from enum import Enum
+from typing import Generic, List, Optional, Tuple, Type, TypeVar, Union
+
+logger = logging.getLogger(__name__)
+
+
+class LogicalOperator(Enum):
+    AND = "AND"
+    OR = "OR"
+
+
+class SearchField:
+    def __init__(self, field_name: str):
+        self.field_name = field_name
+
+    def get_search_value(self, value: str) -> str:
+        return value
+
+    def __str__(self) -> str:
+        return self.field_name
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    @classmethod
+    def from_string_field(cls, field_name: str) -> "SearchField":
+        return cls(field_name)
+
+
+class QueryNode:
+    def __init__(self, operator: Optional[LogicalOperator] = None):
+        self.operator = operator
+        self.children: List[Union[QueryNode, str]] = []
+
+    def add_child(self, child: Union["QueryNode", str]) -> None:
+        self.children.append(child)
+
+    def build(self) -> str:
+        if not self.children:
+            return ""
+
+        if self.operator is None:
+            return (
+                self.children[0]
+                if isinstance(self.children[0], str)
+                else self.children[0].build()
+            )
+
+        child_queries = []
+        for child in self.children:
+            if isinstance(child, str):
+                child_queries.append(child)
+            else:
+                child_queries.append(child.build())
+
+        joined_queries = f" {self.operator.value} ".join(child_queries)
+        return f"({joined_queries})" if len(child_queries) > 1 else joined_queries
+
+
+class ElasticsearchQueryBuilder:
+    SPECIAL_CHARACTERS = r'+-=&|><!(){}[]^"~*?:\/'
+
+    def __init__(self, operator: LogicalOperator = LogicalOperator.AND) -> None:
+        self.root = QueryNode(operator=operator)
+
+    @classmethod
+    def escape_special_characters(cls, value: str) -> str:
+        """
+        Escape special characters in the search term.
+        """
+        return re.sub(f"([{re.escape(cls.SPECIAL_CHARACTERS)}])", r"\\\1", value)
+
+    def _create_term(
+        self, field: SearchField, value: str, is_exact: bool = False
+    ) -> str:
+        escaped_value = self.escape_special_characters(field.get_search_value(value))
+        field_name: str = field.field_name
+        if is_exact:
+            return f'{field_name}:"{escaped_value}"'
+        return f"{field_name}:{escaped_value}"
+
+    def add_field_match(
+        self, field: SearchField, value: str, is_exact: bool = True
+    ) -> "ElasticsearchQueryBuilder":
+        term = self._create_term(field, value, is_exact)
+        self.root.add_child(term)
+        return self
+
+    def add_field_not_match(
+        self, field: SearchField, value: str, is_exact: bool = True
+    ) -> "ElasticsearchQueryBuilder":
+        term = f"-{self._create_term(field, value, is_exact)}"
+        self.root.add_child(term)
+        return self
+
+    def add_range(
+        self,
+        field: str,
+        min_value: Optional[str] = None,
+        max_value: Optional[str] = None,
+        include_min: bool = True,
+        include_max: bool = True,
+    ) -> "ElasticsearchQueryBuilder":
+        min_bracket = "[" if include_min else "{"
+        max_bracket = "]" if include_max else "}"
+        min_val = min_value if min_value is not None else "*"
+        max_val = max_value if max_value is not None else "*"
+        range_query = f"{field}:{min_bracket}{min_val} TO {max_val}{max_bracket}"
+        self.root.add_child(range_query)
+        return self
+
+    def add_wildcard(self, field: str, pattern: str) -> "ElasticsearchQueryBuilder":
+        wildcard_query = f"{field}:{pattern}"
+        self.root.add_child(wildcard_query)
+        return self
+
+    def add_fuzzy(
+        self, field: str, value: str, fuzziness: int = 2
+    ) -> "ElasticsearchQueryBuilder":
+        fuzzy_query = f"{field}:{value}~{fuzziness}"
+        self.root.add_child(fuzzy_query)
+        return self
+
+    def add_boost(
+        self, field: str, value: str, boost: float
+    ) -> "ElasticsearchQueryBuilder":
+        boosted_query = f"{field}:{value}^{boost}"
+        self.root.add_child(boosted_query)
+        return self
+
+    def group(self, operator: LogicalOperator) -> "QueryGroup":
+        return QueryGroup(self, operator)
+
+    def build(self) -> str:
+        return self.root.build()
+
+
+class QueryGroup:
+    def __init__(self, parent: ElasticsearchQueryBuilder, operator: LogicalOperator):
+        self.parent = parent
+        self.node = QueryNode(operator)
+        self.parent.root.add_child(self.node)
+
+    def add_field_match(
+        self, field: Union[str, SearchField], value: str, is_exact: bool = True
+    ) -> "QueryGroup":
+        if isinstance(field, str):
+            field = SearchField.from_string_field(field)
+        term = self.parent._create_term(field, value, is_exact)
+        self.node.add_child(term)
+        return self
+
+    def add_field_not_match(
+        self, field: Union[str, SearchField], value: str, is_exact: bool = True
+    ) -> "QueryGroup":
+        if isinstance(field, str):
+            field = SearchField.from_string_field(field)
+        term = f"-{self.parent._create_term(field, value, is_exact)}"
+        self.node.add_child(term)
+        return self
+
+    def add_range(
+        self,
+        field: str,
+        min_value: Optional[str] = None,
+        max_value: Optional[str] = None,
+        include_min: bool = True,
+        include_max: bool = True,
+    ) -> "QueryGroup":
+        min_bracket = "[" if include_min else "{"
+        max_bracket = "]" if include_max else "}"
+        min_val = min_value if min_value is not None else "*"
+        max_val = max_value if max_value is not None else "*"
+        range_query = f"{field}:{min_bracket}{min_val} TO {max_val}{max_bracket}"
+        self.node.add_child(range_query)
+        return self
+
+    def add_wildcard(self, field: str, pattern: str) -> "QueryGroup":
+        wildcard_query = f"{field}:{pattern}"
+        self.node.add_child(wildcard_query)
+        return self
+
+    def add_fuzzy(self, field: str, value: str, fuzziness: int = 2) -> "QueryGroup":
+        fuzzy_query = f"{field}:{value}~{fuzziness}"
+        self.node.add_child(fuzzy_query)
+        return self
+
+    def add_boost(self, field: str, value: str, boost: float) -> "QueryGroup":
+        boosted_query = f"{field}:{value}^{boost}"
+        self.node.add_child(boosted_query)
+        return self
+
+    def group(self, operator: LogicalOperator) -> "QueryGroup":
+        new_group = QueryGroup(self.parent, operator)
+        self.node.add_child(new_group.node)
+        return new_group
+
+    def end(self) -> ElasticsearchQueryBuilder:
+        return self.parent
+
+
+SF = TypeVar("SF", bound=SearchField)
+
+
+class ElasticDocumentQuery(Generic[SF]):
+    def __init__(self) -> None:
+        self.query_builder = ElasticsearchQueryBuilder()
+
+    @classmethod
+    def create_from(
+        cls: Type["ElasticDocumentQuery[SF]"],
+        *args: Tuple[Union[str, SF], str],
+    ) -> "ElasticDocumentQuery[SF]":
+        instance = cls()
+        for arg in args:
+            if isinstance(arg, SearchField):
+                # If the value is empty, we treat it as a wildcard search
+                logger.info(f"Adding wildcard search for field {arg}")
+                instance.add_wildcard(arg, "*")
+            elif isinstance(arg, tuple) and len(arg) == 2:
+                field, value = arg
+                assert isinstance(value, str)
+                if isinstance(field, SearchField):
+                    instance.add_field_match(field, value)
+                elif isinstance(field, str):
+                    instance.add_field_match(
+                        SearchField.from_string_field(field), value
+                    )
+                else:
+                    raise ValueError("Invalid field type {}".format(type(field)))
+        return instance
+
+    def add_field_match(
+        self, field: Union[str, SearchField], value: str, is_exact: bool = True
+    ) -> "ElasticDocumentQuery":
+        if isinstance(field, str):
+            field = SearchField.from_string_field(field)
+        self.query_builder.add_field_match(field, value, is_exact)
+        return self
+
+    def add_field_not_match(
+        self, field: SearchField, value: str, is_exact: bool = True
+    ) -> "ElasticDocumentQuery":
+        self.query_builder.add_field_not_match(field, value, is_exact)
+        return self
+
+    def add_range(
+        self,
+        field: SearchField,
+        min_value: Optional[str] = None,
+        max_value: Optional[str] = None,
+        include_min: bool = True,
+        include_max: bool = True,
+    ) -> "ElasticDocumentQuery":
+        field_name: str = field.field_name  # type: ignore
+        self.query_builder.add_range(
+            field_name, min_value, max_value, include_min, include_max
+        )
+        return self
+
+    def add_wildcard(self, field: SearchField, pattern: str) -> "ElasticDocumentQuery":
+        field_name: str = field.field_name  # type: ignore
+        self.query_builder.add_wildcard(field_name, pattern)
+        return self
+
+    def add_fuzzy(
+        self, field: SearchField, value: str, fuzziness: int = 2
+    ) -> "ElasticDocumentQuery":
+        field_name: str = field.field_name  # type: ignore
+        self.query_builder.add_fuzzy(field_name, value, fuzziness)
+        return self
+
+    def add_boost(
+        self, field: SearchField, value: str, boost: float
+    ) -> "ElasticDocumentQuery":
+        self.query_builder.add_boost(field.field_name, value, boost)
+        return self
+
+    def group(self, operator: LogicalOperator) -> QueryGroup:
+        return self.query_builder.group(operator)
+
+    def build(self) -> str:
+        return self.query_builder.build()
diff --git a/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py b/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py
deleted file mode 100644
index 5a8802c7a0a49..0000000000000
--- a/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py
+++ /dev/null
@@ -1,160 +0,0 @@
-import contextlib
-import logging
-import re
-import unittest
-import unittest.mock
-from typing import Dict, List, Optional, Set
-
-from sqllineage.core.holders import Column, SQLLineageHolder
-from sqllineage.exceptions import SQLLineageException
-
-from datahub.utilities.sql_parser_base import SQLParser, SqlParserException
-
-with contextlib.suppress(ImportError):
-    import sqlparse
-    from networkx import DiGraph
-    from sqllineage.core import LineageAnalyzer
-
-    import datahub.utilities.sqllineage_patch
-logger = logging.getLogger(__name__)
-
-
-class SqlLineageSQLParserImpl(SQLParser):
-    _DATE_SWAP_TOKEN = "__d_a_t_e"
-    _HOUR_SWAP_TOKEN = "__h_o_u_r"
-    _TIMESTAMP_SWAP_TOKEN = "__t_i_m_e_s_t_a_m_p"
-    _DATA_SWAP_TOKEN = "__d_a_t_a"
-    _ADMIN_SWAP_TOKEN = "__a_d_m_i_n"
-    _MYVIEW_SQL_TABLE_NAME_TOKEN = "__my_view__.__sql_table_name__"
-    _MYVIEW_LOOKER_TOKEN = "my_view.SQL_TABLE_NAME"
-
-    def __init__(self, sql_query: str, use_raw_names: bool = False) -> None:
-        super().__init__(sql_query)
-        original_sql_query = sql_query
-        self._use_raw_names = use_raw_names
-
-        # SqlLineageParser makes mistakes on lateral flatten queries, use the prefix
-        if "lateral flatten" in sql_query:
-            sql_query = sql_query[: sql_query.find("lateral flatten")]
-
-        # Replace reserved words that break SqlLineageParser
-        self.token_to_original: Dict[str, str] = {
-            self._DATE_SWAP_TOKEN: "date",
-            self._HOUR_SWAP_TOKEN: "hour",
-            self._TIMESTAMP_SWAP_TOKEN: "timestamp",
-            self._DATA_SWAP_TOKEN: "data",
-            self._ADMIN_SWAP_TOKEN: "admin",
-        }
-        for replacement, original in self.token_to_original.items():
-            # Replace original tokens with replacement. Since table and column name can contain a hyphen('-'),
-            # also prevent original tokens appearing as part of these names with a hyphen from getting substituted.
-            sql_query = re.sub(
-                rf"((?<!-)\b{original}\b)(?!-)",
-                rf"{replacement}",
-                sql_query,
-                flags=re.IGNORECASE,
-            )
-
-        # SqlLineageParser lowercarese tablenames and we need to replace Looker specific token which should be uppercased
-        sql_query = re.sub(
-            rf"(\${{{self._MYVIEW_LOOKER_TOKEN}}})",
-            rf"{self._MYVIEW_SQL_TABLE_NAME_TOKEN}",
-            sql_query,
-        )
-
-        # SqlLineageParser does not handle "encode" directives well. Remove them
-        sql_query = re.sub(r"\sencode [a-zA-Z]*", "", sql_query, flags=re.IGNORECASE)
-
-        # Replace lookml templates with the variable otherwise sqlparse can't parse ${
-        sql_query = re.sub(r"(\${)(.+)(})", r"\2", sql_query)
-        if sql_query != original_sql_query:
-            logger.debug(f"Rewrote original query {original_sql_query} as {sql_query}")
-
-        self._sql = sql_query
-        self._stmt_holders: Optional[List[LineageAnalyzer]] = None
-        self._sql_holder: Optional[SQLLineageHolder] = None
-        try:
-            self._stmt = [
-                s
-                for s in sqlparse.parse(
-                    # first apply sqlparser formatting just to get rid of comments, which cause
-                    # inconsistencies in parsing output
-                    sqlparse.format(
-                        self._sql.strip(),
-                        strip_comments=True,
-                        use_space_around_operators=True,
-                    ),
-                )
-                if s.token_first(skip_cm=True)
-            ]
-
-            with unittest.mock.patch(
-                "sqllineage.core.handlers.source.SourceHandler.end_of_query_cleanup",
-                datahub.utilities.sqllineage_patch.end_of_query_cleanup_patch,
-            ):
-                with unittest.mock.patch(
-                    "sqllineage.core.holders.SubQueryLineageHolder.add_column_lineage",
-                    datahub.utilities.sqllineage_patch.add_column_lineage_patch,
-                ):
-                    self._stmt_holders = [
-                        LineageAnalyzer().analyze(stmt) for stmt in self._stmt
-                    ]
-                    self._sql_holder = SQLLineageHolder.of(*self._stmt_holders)
-        except SQLLineageException as e:
-            raise SqlParserException(
-                f"SQL lineage analyzer error '{e}' for query: '{self._sql}"
-            ) from e
-
-    def get_tables(self) -> List[str]:
-        result: List[str] = []
-        if self._sql_holder is None:
-            logger.error("sql holder not present so cannot get tables")
-            return result
-        for table in self._sql_holder.source_tables:
-            table_normalized = re.sub(
-                r"^<default>.",
-                "",
-                (
-                    str(table)
-                    if not self._use_raw_names
-                    else f"{table.schema.raw_name}.{table.raw_name}"
-                ),
-            )
-            result.append(str(table_normalized))
-
-        # We need to revert TOKEN replacements
-        for token, replacement in self.token_to_original.items():
-            result = [replacement if c == token else c for c in result]
-        result = [
-            self._MYVIEW_LOOKER_TOKEN if c == self._MYVIEW_SQL_TABLE_NAME_TOKEN else c
-            for c in result
-        ]
-
-        # Sort tables to make the list deterministic
-        result.sort()
-
-        return result
-
-    def get_columns(self) -> List[str]:
-        if self._sql_holder is None:
-            raise SqlParserException("sql holder not present so cannot get columns")
-        graph: DiGraph = self._sql_holder.graph  # For mypy attribute checking
-        column_nodes = [n for n in graph.nodes if isinstance(n, Column)]
-        column_graph = graph.subgraph(column_nodes)
-
-        target_columns = {column for column, deg in column_graph.out_degree if deg == 0}
-
-        result: Set[str] = set()
-        for column in target_columns:
-            # Let's drop all the count(*) and similard columns which are expression actually if it does not have an alias
-            if not any(ele in column.raw_name for ele in ["*", "(", ")"]):
-                result.add(str(column.raw_name))
-
-        # Reverting back all the previously renamed words which confuses the parser
-        result = {"date" if c == self._DATE_SWAP_TOKEN else c for c in result}
-        result = {
-            "timestamp" if c == self._TIMESTAMP_SWAP_TOKEN else c for c in list(result)
-        }
-
-        # swap back renamed date column
-        return list(result)
diff --git a/metadata-ingestion/src/datahub/utilities/sql_parser.py b/metadata-ingestion/src/datahub/utilities/sql_parser.py
deleted file mode 100644
index 61693b52b350f..0000000000000
--- a/metadata-ingestion/src/datahub/utilities/sql_parser.py
+++ /dev/null
@@ -1,94 +0,0 @@
-import logging
-import multiprocessing
-import traceback
-from multiprocessing import Process, Queue
-from typing import Any, List, Optional, Tuple
-
-from datahub.utilities.sql_lineage_parser_impl import SqlLineageSQLParserImpl
-from datahub.utilities.sql_parser_base import SQLParser
-
-logger = logging.getLogger(__name__)
-
-
-def sql_lineage_parser_impl_func_wrapper(
-    queue: Optional[multiprocessing.Queue], sql_query: str, use_raw_names: bool = False
-) -> Optional[Tuple[List[str], List[str], Any]]:
-    """
-    The wrapper function that computes the tables and columns using the SqlLineageSQLParserImpl
-    and puts the results on the shared IPC queue. This is used to isolate SqlLineageSQLParserImpl
-    functionality in a separate process, and hence protect our sources from memory leaks originating in
-    the sqllineage module.
-    :param queue: The shared IPC queue on to which the results will be put.
-    :param sql_query: The SQL query to extract the tables & columns from.
-    :param use_raw_names: Parameter used to ignore sqllineage's default lowercasing.
-    :return: None.
-    """
-    exception_details: Optional[Tuple[BaseException, str]] = None
-    tables: List[str] = []
-    columns: List[str] = []
-    try:
-        parser = SqlLineageSQLParserImpl(sql_query, use_raw_names)
-        tables = parser.get_tables()
-        columns = parser.get_columns()
-    except BaseException as e:
-        exc_msg = traceback.format_exc()
-        exception_details = (e, exc_msg)
-        logger.debug(exc_msg)
-
-    if queue is not None:
-        queue.put((tables, columns, exception_details))
-        return None
-    else:
-        return (tables, columns, exception_details)
-
-
-class SqlLineageSQLParser(SQLParser):
-    def __init__(
-        self,
-        sql_query: str,
-        use_external_process: bool = True,
-        use_raw_names: bool = False,
-    ) -> None:
-        super().__init__(sql_query, use_external_process)
-        if use_external_process:
-            self.tables, self.columns = self._get_tables_columns_process_wrapped(
-                sql_query, use_raw_names
-            )
-        else:
-            return_tuple = sql_lineage_parser_impl_func_wrapper(
-                None, sql_query, use_raw_names
-            )
-            if return_tuple is not None:
-                (
-                    self.tables,
-                    self.columns,
-                    some_exception,
-                ) = return_tuple
-
-    @staticmethod
-    def _get_tables_columns_process_wrapped(
-        sql_query: str, use_raw_names: bool = False
-    ) -> Tuple[List[str], List[str]]:
-        # Invoke sql_lineage_parser_impl_func_wrapper in a separate process to avoid
-        # memory leaks from sqllineage module used by SqlLineageSQLParserImpl. This will help
-        # shield our sources like lookml & redash, that need to parse a large number of SQL statements,
-        # from causing significant memory leaks in the datahub cli during ingestion.
-        queue: multiprocessing.Queue = Queue()
-        process: multiprocessing.Process = Process(
-            target=sql_lineage_parser_impl_func_wrapper,
-            args=(queue, sql_query, use_raw_names),
-        )
-        process.start()
-        tables, columns, exception_details = queue.get(block=True)
-        if exception_details is not None:
-            raise exception_details[0](f"Sub-process exception: {exception_details[1]}")
-        return tables, columns
-
-    def get_tables(self) -> List[str]:
-        return self.tables
-
-    def get_columns(self) -> List[str]:
-        return self.columns
-
-
-DefaultSQLParser = SqlLineageSQLParser
diff --git a/metadata-ingestion/src/datahub/utilities/sql_parser_base.py b/metadata-ingestion/src/datahub/utilities/sql_parser_base.py
deleted file mode 100644
index 8fd5dfaf4978d..0000000000000
--- a/metadata-ingestion/src/datahub/utilities/sql_parser_base.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from abc import ABCMeta, abstractmethod
-from typing import List
-
-
-class SqlParserException(Exception):
-    """Raised when sql parser fails"""
-
-    pass
-
-
-class SQLParser(metaclass=ABCMeta):
-    def __init__(self, sql_query: str, use_external_process: bool = True) -> None:
-        self._sql_query = sql_query
-
-    @abstractmethod
-    def get_tables(self) -> List[str]:
-        pass
-
-    @abstractmethod
-    def get_columns(self) -> List[str]:
-        pass
diff --git a/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py b/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py
index 216fa155035d3..ab8987a7d2e8b 100644
--- a/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py
+++ b/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py
@@ -19,7 +19,6 @@ def process(
         args_list: Iterable[Tuple[Any, ...]],
         max_workers: int,
     ) -> Generator[T, None, None]:
-
         out_q: queue.Queue[T] = queue.Queue()
 
         def _worker_wrapper(
@@ -46,7 +45,6 @@ def _worker_wrapper(
                 futures = [f for f in futures if not f.done()]
                 if not futures:
                     break
-
         # Yield the remaining work units. This theoretically should not happen, but adding it just in case.
         while not out_q.empty():
             yield out_q.get_nowait()
diff --git a/metadata-ingestion/src/datahub/utilities/threading_timeout.py b/metadata-ingestion/src/datahub/utilities/threading_timeout.py
new file mode 100644
index 0000000000000..e2caf57ad2116
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/threading_timeout.py
@@ -0,0 +1,42 @@
+import contextlib
+import functools
+import platform
+from typing import ContextManager
+
+from stopit import ThreadingTimeout as _ThreadingTimeout, TimeoutException
+
+__all__ = ["threading_timeout", "TimeoutException"]
+
+
+@functools.lru_cache(maxsize=1)
+def _is_cpython() -> bool:
+    """Check if we're running on CPython."""
+    return platform.python_implementation() == "CPython"
+
+
+def threading_timeout(timeout: float) -> ContextManager[None]:
+    """A timeout context manager that uses stopit's ThreadingTimeout underneath.
+
+    This is only supported on CPython.
+    That's because stopit.ThreadingTimeout uses a CPython-internal method to raise
+    an exception (the timeout error) in another thread. See stopit.threadstop.async_raise.
+
+    Reference: https://github.com/glenfant/stopit
+
+    Args:
+        timeout: The timeout in seconds. If <= 0, no timeout is applied.
+
+    Raises:
+        RuntimeError: If the timeout is not supported on the current Python implementation.
+        TimeoutException: If the timeout is exceeded.
+    """
+
+    if timeout <= 0:
+        return contextlib.nullcontext()
+
+    if not _is_cpython():
+        raise RuntimeError(
+            f"Timeout is only supported on CPython, not {platform.python_implementation()}"
+        )
+
+    return _ThreadingTimeout(timeout, swallow_exc=False)
diff --git a/metadata-ingestion/src/datahub/utilities/unified_diff.py b/metadata-ingestion/src/datahub/utilities/unified_diff.py
new file mode 100644
index 0000000000000..c896fd4df4d8f
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/unified_diff.py
@@ -0,0 +1,236 @@
+import logging
+from dataclasses import dataclass
+from typing import List, Tuple
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+_LOOKAROUND_LINES = 300
+
+# The Python difflib library can generate unified diffs, but it cannot apply them.
+# There weren't any well-maintained and easy-to-use libraries for applying
+# unified diffs, so I wrote my own.
+#
+# My implementation is focused on ensuring correctness, and will throw
+# an exception whenever it detects an issue.
+#
+# Alternatives considered:
+# - diff-match-patch: This was the most promising since it's from Google.
+#       Unfortunately, they deprecated the library in Aug 2024. That may not have
+#       been a dealbreaker, since a somewhat greenfield community fork exists:
+#       https://github.com/dmsnell/diff-match-patch
+#       However, there's also a long-standing bug in the library around the
+#       handling of line breaks when parsing diffs. See:
+#       https://github.com/google/diff-match-patch/issues/157
+# - python-patch: Seems abandoned.
+# - patch-ng: Fork of python-patch, but mainly targeted at applying patches to trees.
+#       It did not have simple "apply patch to string" abstractions.
+# - unidiff: Parses diffs, but cannot apply them.
+
+
+class InvalidDiffError(Exception):
+    pass
+
+
+class DiffApplyError(Exception):
+    pass
+
+
+@dataclass
+class Hunk:
+    source_start: int
+    source_lines: int
+    target_start: int
+    target_lines: int
+    lines: List[Tuple[str, str]]
+
+
+def parse_patch(patch_text: str) -> List[Hunk]:
+    """
+    Parses a unified diff patch into a list of Hunk objects.
+
+    Args:
+        patch_text: Unified diff format patch text
+
+    Returns:
+        List of parsed Hunk objects
+
+    Raises:
+        InvalidDiffError: If the patch is in an invalid format
+    """
+    hunks = []
+    patch_lines = patch_text.splitlines()
+    i = 0
+
+    while i < len(patch_lines):
+        line = patch_lines[i]
+
+        if line.startswith("@@"):
+            try:
+                header_parts = line.split()
+                if len(header_parts) < 3:
+                    raise ValueError(f"Invalid hunk header format: {line}")
+
+                source_changes, target_changes = header_parts[1:3]
+                source_start, source_lines = map(int, source_changes[1:].split(","))
+                target_start, target_lines = map(int, target_changes[1:].split(","))
+
+                hunk = Hunk(source_start, source_lines, target_start, target_lines, [])
+                i += 1
+
+                while i < len(patch_lines) and not patch_lines[i].startswith("@@"):
+                    hunk_line = patch_lines[i]
+                    if hunk_line:
+                        hunk.lines.append((hunk_line[0], hunk_line[1:]))
+                    else:
+                        # Fully empty lines usually means an empty context line that was
+                        # trimmed by trailing whitespace removal.
+                        hunk.lines.append((" ", ""))
+                    i += 1
+
+                hunks.append(hunk)
+            except (IndexError, ValueError) as e:
+                raise InvalidDiffError(f"Failed to parse hunk: {str(e)}") from e
+        else:
+            raise InvalidDiffError(f"Invalid line format: {line}")
+
+    return hunks
+
+
+def find_hunk_start(source_lines: List[str], hunk: Hunk) -> int:
+    """
+    Finds the actual starting line of a hunk in the source lines.
+
+    Args:
+        source_lines: The original source lines
+        hunk: The hunk to locate
+
+    Returns:
+        The actual line number where the hunk starts
+
+    Raises:
+        DiffApplyError: If the hunk's context cannot be found in the source lines
+    """
+
+    # Extract context lines from the hunk, stopping at the first non-context line
+    context_lines = []
+    for prefix, line in hunk.lines:
+        if prefix == " ":
+            context_lines.append(line)
+        else:
+            break
+
+    if not context_lines:
+        logger.debug("No context lines found in hunk.")
+        return hunk.source_start - 1  # Default to the original start if no context
+
+    logger.debug(
+        f"Searching for {len(context_lines)} context lines, starting with {context_lines[0]}"
+    )
+
+    # Define the range to search for the context lines
+    search_start = max(0, hunk.source_start - _LOOKAROUND_LINES)
+    search_end = min(len(source_lines), hunk.source_start + _LOOKAROUND_LINES)
+
+    # Iterate over the possible starting positions in the source lines
+    for i in range(search_start, search_end):
+        # Check if the context lines match the source lines starting at position i
+        match = True
+        for j, context_line in enumerate(context_lines):
+            if (i + j >= len(source_lines)) or source_lines[i + j] != context_line:
+                match = False
+                break
+        if match:
+            # logger.debug(f"Context match found at line: {i}")
+            return i
+
+    logger.debug(f"Could not find match for hunk context lines: {context_lines}")
+    raise DiffApplyError("Could not find match for hunk context.")
+
+
+def apply_hunk(result_lines: List[str], hunk: Hunk, hunk_index: int) -> None:
+    """
+    Applies a single hunk to the result lines.
+
+    Args:
+        result_lines: The current state of the patched file
+        hunk: The hunk to apply
+        hunk_index: The index of the hunk (for logging purposes)
+
+    Raises:
+        DiffApplyError: If the hunk cannot be applied correctly
+    """
+    current_line = find_hunk_start(result_lines, hunk)
+    logger.debug(f"Hunk {hunk_index + 1} start line: {current_line}")
+
+    for line_index, (prefix, content) in enumerate(hunk.lines):
+        # logger.debug(f"Processing line {line_index + 1} of hunk {hunk_index + 1}")
+        # logger.debug(f"Current line: {current_line}, Total lines: {len(result_lines)}")
+        # logger.debug(f"Prefix: {prefix}, Content: {content}")
+
+        if current_line >= len(result_lines):
+            logger.debug(f"Reached end of file while applying hunk {hunk_index + 1}")
+            while line_index < len(hunk.lines) and hunk.lines[line_index][0] == "+":
+                result_lines.append(hunk.lines[line_index][1])
+                line_index += 1
+
+            # If there's context or deletions past the end of the file, that's an error.
+            if line_index < len(hunk.lines):
+                raise DiffApplyError(
+                    f"Found context or deletions after end of file in hunk {hunk_index + 1}"
+                )
+            break
+
+        if prefix == "-":
+            if result_lines[current_line].strip() != content.strip():
+                raise DiffApplyError(
+                    f"Removing line that doesn't exactly match. Expected: '{content.strip()}', Found: '{result_lines[current_line].strip()}'"
+                )
+            result_lines.pop(current_line)
+        elif prefix == "+":
+            result_lines.insert(current_line, content)
+            current_line += 1
+        elif prefix == " ":
+            if result_lines[current_line].strip() != content.strip():
+                raise DiffApplyError(
+                    f"Context line doesn't exactly match. Expected: '{content.strip()}', Found: '{result_lines[current_line].strip()}'"
+                )
+            current_line += 1
+        else:
+            raise DiffApplyError(
+                f"Invalid line prefix '{prefix}' in hunk {hunk_index + 1}, line {line_index + 1}"
+            )
+
+
+def apply_diff(source: str, patch_text: str) -> str:
+    """
+    Applies a unified diff patch to source text and returns the patched result.
+
+    Args:
+        source: Original source text to be patched
+        patch_text: Unified diff format patch text (with @@ markers and hunks)
+
+    Returns:
+        The patched text result
+
+    Raises:
+        InvalidDiffError: If the patch is in an invalid format
+        DiffApplyError: If the patch cannot be applied correctly
+    """
+
+    # logger.debug(f"Original source:\n{source}")
+    # logger.debug(f"Patch text:\n{patch_text}")
+
+    hunks = parse_patch(patch_text)
+    logger.debug(f"Parsed into {len(hunks)} hunks")
+
+    source_lines = source.splitlines()
+    result_lines = source_lines.copy()
+
+    for hunk_index, hunk in enumerate(hunks):
+        logger.debug(f"Processing hunk {hunk_index + 1}")
+        apply_hunk(result_lines, hunk, hunk_index)
+
+    result = "\n".join(result_lines) + "\n"
+    # logger.debug(f"Patched result:\n{result}")
+    return result
diff --git a/metadata-ingestion/src/datahub/utilities/urn_encoder.py b/metadata-ingestion/src/datahub/utilities/urn_encoder.py
index 88c0a128b8e46..4f19eeff3e70f 100644
--- a/metadata-ingestion/src/datahub/utilities/urn_encoder.py
+++ b/metadata-ingestion/src/datahub/utilities/urn_encoder.py
@@ -4,7 +4,8 @@
 # NOTE: Frontend relies on encoding these three characters. Specifically, we decode and encode schema fields for column level lineage.
 # If this changes, make appropriate changes to datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts
 # We also rely on encoding these exact three characters when generating schemaField urns in our graphQL layer. Update SchemaFieldUtils if this changes.
-RESERVED_CHARS = {",", "(", ")"}
+# Also see https://datahubproject.io/docs/what/urn/#restrictions
+RESERVED_CHARS = {",", "(", ")", "␟"}
 RESERVED_CHARS_EXTENDED = RESERVED_CHARS.union({"%"})
 
 
diff --git a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py
index 1b50d4b2fe810..7dadd16fb7f1c 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py
@@ -200,7 +200,7 @@ def get_entity_id_as_string(self) -> str:
     @classmethod
     @deprecated(reason="no longer needed")
     def validate(cls, urn_str: str) -> None:
-        Urn.create_from_string(urn_str)
+        Urn.from_string(urn_str)
 
     @staticmethod
     def url_encode(urn: str) -> str:
diff --git a/metadata-ingestion/src/datahub/utilities/urns/corp_group_urn.py b/metadata-ingestion/src/datahub/utilities/urns/corp_group_urn.py
index 37c1076925945..577f90215a635 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/corp_group_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/corp_group_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import CorpGroupUrn  # noqa: F401
+from datahub.metadata.urns import CorpGroupUrn
+
+__all__ = ["CorpGroupUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/corpuser_urn.py b/metadata-ingestion/src/datahub/utilities/urns/corpuser_urn.py
index 5f9ecf65951b9..8acb86be00f6c 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/corpuser_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/corpuser_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import CorpUserUrn as CorpuserUrn  # noqa: F401
+from datahub.metadata.urns import CorpUserUrn as CorpuserUrn
+
+__all__ = ["CorpuserUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/data_flow_urn.py b/metadata-ingestion/src/datahub/utilities/urns/data_flow_urn.py
index 5b2b45927c339..3508ae5c4a349 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/data_flow_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/data_flow_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import DataFlowUrn  # noqa: F401
+from datahub.metadata.urns import DataFlowUrn
+
+__all__ = ["DataFlowUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/data_job_urn.py b/metadata-ingestion/src/datahub/utilities/urns/data_job_urn.py
index 53e3419ee7ecb..d003b6c6ad7a8 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/data_job_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/data_job_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import DataJobUrn  # noqa: F401
+from datahub.metadata.urns import DataJobUrn
+
+__all__ = ["DataJobUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/data_platform_urn.py b/metadata-ingestion/src/datahub/utilities/urns/data_platform_urn.py
index 9d37e38f256e7..51e013e715d4f 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/data_platform_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/data_platform_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import DataPlatformUrn  # noqa: F401
+from datahub.metadata.urns import DataPlatformUrn
+
+__all__ = ["DataPlatformUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/data_process_instance_urn.py b/metadata-ingestion/src/datahub/utilities/urns/data_process_instance_urn.py
index df6ba797d069c..22e6b36c5f7ae 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/data_process_instance_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/data_process_instance_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import DataProcessInstanceUrn  # noqa: F401
+from datahub.metadata.urns import DataProcessInstanceUrn
+
+__all__ = ["DataProcessInstanceUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py b/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py
index 6078ffefc03d8..1652e17059995 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import DatasetUrn  # noqa: F401
+from datahub.metadata.urns import DatasetUrn
+
+__all__ = ["DatasetUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/domain_urn.py b/metadata-ingestion/src/datahub/utilities/urns/domain_urn.py
index 442a6b27729bb..242a3d8228320 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/domain_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/domain_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import DomainUrn  # noqa: F401
+from datahub.metadata.urns import DomainUrn
+
+__all__ = ["DomainUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/notebook_urn.py b/metadata-ingestion/src/datahub/utilities/urns/notebook_urn.py
index 60a4f5396aa46..f9b861d7f0852 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/notebook_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/notebook_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import NotebookUrn  # noqa: F401
+from datahub.metadata.urns import NotebookUrn
+
+__all__ = ["NotebookUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py
index 5bd36a0656d99..748d5519f1477 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py
@@ -1,5 +1,7 @@
-from datahub.metadata.urns import StructuredPropertyUrn  # noqa: F401
+from datahub.metadata.urns import StructuredPropertyUrn
+
+__all__ = ["StructuredPropertyUrn", "make_structured_property_urn"]
 
 
 def make_structured_property_urn(structured_property_id: str) -> str:
-    return str(StructuredPropertyUrn.create_from_string(structured_property_id))
+    return str(StructuredPropertyUrn.from_string(structured_property_id))
diff --git a/metadata-ingestion/src/datahub/utilities/urns/tag_urn.py b/metadata-ingestion/src/datahub/utilities/urns/tag_urn.py
index 0ac632ee40a01..f66d56a745a96 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/tag_urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/tag_urn.py
@@ -1 +1,3 @@
-from datahub.metadata.urns import TagUrn  # noqa: F401
+from datahub.metadata.urns import TagUrn
+
+__all__ = ["TagUrn"]
diff --git a/metadata-ingestion/src/datahub/utilities/urns/urn.py b/metadata-ingestion/src/datahub/utilities/urns/urn.py
index 2e5cebfd0e8f5..2ded2d4d9b32c 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/urn.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/urn.py
@@ -1,4 +1,6 @@
-from datahub.metadata.urns import Urn  # noqa: F401
+from datahub.metadata.urns import Urn
+
+__all__ = ["Urn", "guess_entity_type"]
 
 
 def guess_entity_type(urn: str) -> str:
diff --git a/metadata-ingestion/tests/conftest.py b/metadata-ingestion/tests/conftest.py
index d0716e34ee2b6..4685faabfcb28 100644
--- a/metadata-ingestion/tests/conftest.py
+++ b/metadata-ingestion/tests/conftest.py
@@ -7,6 +7,7 @@
 import pytest
 
 os.environ["DATAHUB_SUPPRESS_LOGGING_MANAGER"] = "1"
+os.environ["DATAHUB_TEST_MODE"] = "1"
 
 # Enable debug logging.
 logging.getLogger().setLevel(logging.DEBUG)
@@ -25,7 +26,10 @@
     docker_compose_command,
     docker_compose_runner,
 )
-from tests.test_helpers.state_helpers import mock_datahub_graph  # noqa: F401,E402
+from tests.test_helpers.state_helpers import (  # noqa: F401,E402
+    mock_datahub_graph,
+    mock_datahub_graph_instance,
+)
 
 try:
     # See https://github.com/spulec/freezegun/issues/98#issuecomment-590553475.
diff --git a/metadata-ingestion/tests/integration/azure_ad/test_azure_ad.py b/metadata-ingestion/tests/integration/azure_ad/test_azure_ad.py
index 7005bc2e4411b..024bb62bbe9ce 100644
--- a/metadata-ingestion/tests/integration/azure_ad/test_azure_ad.py
+++ b/metadata-ingestion/tests/integration/azure_ad/test_azure_ad.py
@@ -68,7 +68,6 @@ def run_ingest(
         "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph",
         mock_datahub_graph,
     ) as mock_checkpoint:
-
         mock_checkpoint.return_value = mock_datahub_graph
 
         mocked_functions_reference(
diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json
index 02660f0fae08e..5e091596cc0f7 100644
--- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json
+++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json
@@ -112,6 +112,26 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": [
+                {
+                    "tag": "urn:li:tag:priority:medium:test"
+                }
+            ]
+    }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0",
@@ -199,6 +219,180 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "BigQueryLabelInfo",
+            "primaryKey": "priority/high",
+            "secondaryKeys": [
+                "urn:li:tag:priority:high"
+            ],
+            "value": {
+                "blob": "{\"datahub_urn\": \"urn:li:tag:priority:high\", \"managed_by_datahub\": false, \"key\": \"priority\", \"value\": \"high\"}",
+                "contentType": "JSON",
+                "schemaType": "JSON",
+                "schemaRef": "BigQueryLabelInfo"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:bigquery"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:7fbbf79fb726422dc2434222a8e30630",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "BigQueryLabelInfo",
+            "primaryKey": "priority/medium:test",
+            "secondaryKeys": [
+                "urn:li:tag:priority:medium:test"
+            ],
+            "value": {
+                "blob": "{\"datahub_urn\": \"urn:li:tag:priority:medium:test\", \"managed_by_datahub\": false, \"key\": \"priority\", \"value\": \"medium:test\"}",
+                "contentType": "JSON",
+                "schemaType": "JSON",
+                "schemaRef": "BigQueryLabelInfo"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:7fbbf79fb726422dc2434222a8e30630",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:bigquery"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:7fbbf79fb726422dc2434222a8e30630",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "BigQueryLabelInfo",
+            "primaryKey": "purchase",
+            "secondaryKeys": [
+                "urn:li:tag:purchase"
+            ],
+            "value": {
+                "blob": "{\"datahub_urn\": \"urn:li:tag:purchase\", \"managed_by_datahub\": false, \"key\": \"purchase\", \"value\": \"\"}",
+                "contentType": "JSON",
+                "schemaType": "JSON",
+                "schemaRef": "BigQueryLabelInfo"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:bigquery"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)",
@@ -330,6 +524,45 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:7da6409504c5c6444b4ce60b0239b759",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "BigQueryLabelInfo",
+            "primaryKey": "mixedcasetag",
+            "value": {
+                "blob": "{\"datahub_urn\": \"urn:li:tag:MixedCaseTag\", \"managed_by_datahub\": true, \"key\": \"mixedcasetag\", \"value\": \"\"}",
+                "contentType": "JSON",
+                "schemaType": "JSON",
+                "schemaRef": "BigQueryLabelInfo"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:7da6409504c5c6444b4ce60b0239b759",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:bigquery"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)",
@@ -343,6 +576,9 @@
                 },
                 {
                     "tag": "urn:li:tag:purchase"
+                },
+                {
+                    "tag": "urn:li:tag:MixedCaseTag"
                 }
             ]
         }
@@ -353,6 +589,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:7da6409504c5c6444b4ce60b0239b759",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)",
@@ -1067,6 +1319,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:priority:medium:test",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "priority:medium:test"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:purchase",
@@ -1082,5 +1350,21 @@
         "runId": "bigquery-2022_02_03-07_00_00",
         "lastRunId": "no-run-id-provided"
     }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:MixedCaseTag",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "MixedCaseTag"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_queries_mcps_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_queries_mcps_golden.json
index 1b79e8464c05f..9a4881f016960 100644
--- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_queries_mcps_golden.json
+++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_queries_mcps_golden.json
@@ -193,7 +193,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,gcp-staging.smoke_test_db.base_table,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b"
+                    "query": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d"
                 }
             ]
         }
@@ -206,7 +206,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b",
+    "entityUrn": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d",
     "changeType": "UPSERT",
     "aspectName": "queryProperties",
     "aspect": {
@@ -425,7 +425,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b",
+    "entityUrn": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d",
     "changeType": "UPSERT",
     "aspectName": "querySubjects",
     "aspect": {
@@ -508,7 +508,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b",
+    "entityUrn": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
@@ -7890,7 +7890,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b",
+    "entityUrn": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py
index 0ac4e94a5a24f..1f14688636161 100644
--- a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py
+++ b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py
@@ -1,12 +1,16 @@
 import random
 import string
 from datetime import datetime, timezone
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 from unittest.mock import MagicMock, patch
 
 from freezegun import freeze_time
 from google.cloud.bigquery.table import TableListItem
 
+from datahub.api.entities.platformresource.platform_resource import (
+    PlatformResource,
+    PlatformResourceKey,
+)
 from datahub.ingestion.glossary.classifier import (
     ClassificationConfig,
     DynamicTypedClassifierConfig,
@@ -14,6 +18,10 @@
 from datahub.ingestion.glossary.datahub_classifier import DataHubClassifierConfig
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier
 from datahub.ingestion.source.bigquery_v2.bigquery_data_reader import BigQueryDataReader
+from datahub.ingestion.source.bigquery_v2.bigquery_platform_resource_helper import (
+    BigQueryLabelInfo,
+    BigQueryPlatformResourceHelper,
+)
 from datahub.ingestion.source.bigquery_v2.bigquery_schema import (
     BigqueryColumn,
     BigqueryDataset,
@@ -51,10 +59,18 @@ def recipe(mcp_output_path: str, source_config_override: dict = {}) -> dict:
             "type": "bigquery",
             "config": {
                 "project_ids": ["project-id-1"],
+                "credential": {
+                    "project_id": "project-id-1",
+                    "private_key_id": "private_key_id",
+                    "private_key": "private_key",
+                    "client_email": "client_email",
+                    "client_id": "client_id",
+                },
                 "include_usage_statistics": False,
                 "include_table_lineage": True,
                 "include_data_platform_instance": True,
                 "capture_table_label_as_tag": True,
+                "capture_dataset_label_as_tag": True,
                 "classification": ClassificationConfig(
                     enabled=True,
                     classifiers=[
@@ -82,6 +98,7 @@ def recipe(mcp_output_path: str, source_config_override: dict = {}) -> dict:
 @patch.object(BigQuerySchemaApi, "get_datasets_for_project_id")
 @patch.object(BigQuerySchemaApi, "get_columns_for_dataset")
 @patch.object(BigQueryDataReader, "get_sample_data_for_table")
+@patch.object(BigQueryPlatformResourceHelper, "get_platform_resource")
 @patch("google.cloud.bigquery.Client")
 @patch("google.cloud.datacatalog_v1.PolicyTagManagerClient")
 @patch("google.cloud.resourcemanager_v3.ProjectsClient")
@@ -89,6 +106,7 @@ def test_bigquery_v2_ingest(
     client,
     policy_tag_manager_client,
     projects_client,
+    get_platform_resource,
     get_sample_data_for_table,
     get_columns_for_dataset,
     get_datasets_for_project_id,
@@ -104,8 +122,30 @@ def test_bigquery_v2_ingest(
     mcp_output_path = "{}/{}".format(tmp_path, "bigquery_mcp_output.json")
 
     dataset_name = "bigquery-dataset-1"
+
+    def side_effect(*args: Any) -> Optional[PlatformResource]:
+        if args[0].primary_key == "mixedcasetag":
+            return PlatformResource.create(
+                key=PlatformResourceKey(
+                    primary_key="mixedcasetag",
+                    resource_type="BigQueryLabelInfo",
+                    platform="bigquery",
+                ),
+                value=BigQueryLabelInfo(
+                    datahub_urn="urn:li:tag:MixedCaseTag",
+                    managed_by_datahub=True,
+                    key="mixedcasetag",
+                    value="",
+                ),
+            )
+        return None
+
+    get_platform_resource.side_effect = side_effect
     get_datasets_for_project_id.return_value = [
-        BigqueryDataset(name=dataset_name, location="US")
+        # BigqueryDataset(name=dataset_name, location="US")
+        BigqueryDataset(
+            name=dataset_name, location="US", labels={"priority": "medium:test"}
+        )
     ]
 
     table_list_item = TableListItem(
@@ -158,7 +198,8 @@ def test_bigquery_v2_ingest(
         rows_count=None,
         labels={
             "priority": "high",
-            "purchase": "urn_li_encoded_tag_ovzg4otmne5hiylhhjyhk4tdnbqxgzi_",
+            "purchase": "",
+            "mixedcasetag": "",
         },
     )
     get_tables_for_dataset.return_value = iter([bigquery_table])
diff --git a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py
index 9290100b0c521..806779475dea9 100644
--- a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py
+++ b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py
@@ -1,4 +1,5 @@
 import json
+import os
 from datetime import datetime
 from pathlib import Path
 from unittest.mock import patch
@@ -6,7 +7,9 @@
 import pytest
 from freezegun import freeze_time
 
+from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.bigquery_v2.bigquery_queries import (
+    BigQueryQueriesSource,
     BigQueryQueriesSourceReport,
 )
 from datahub.metadata.urns import CorpUserUrn
@@ -44,7 +47,6 @@ def _generate_queries_cached_file(tmp_path: Path, queries_json_path: Path) -> No
 @patch("google.cloud.bigquery.Client")
 @patch("google.cloud.resourcemanager_v3.ProjectsClient")
 def test_queries_ingestion(project_client, client, pytestconfig, monkeypatch, tmp_path):
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/bigquery_v2"
     mcp_golden_path = f"{test_resources_dir}/bigquery_queries_mcps_golden.json"
     mcp_output_path = tmp_path / "bigquery_queries_mcps.json"
@@ -93,3 +95,16 @@ def test_queries_ingestion(project_client, client, pytestconfig, monkeypatch, tm
         output_path=mcp_output_path,
         golden_path=mcp_golden_path,
     )
+
+
+@patch("google.cloud.bigquery.Client")
+@patch("google.cloud.resourcemanager_v3.ProjectsClient")
+def test_source_close_cleans_tmp(projects_client, client, tmp_path):
+    with patch("tempfile.tempdir", str(tmp_path)):
+        source = BigQueryQueriesSource.create(
+            {"project_ids": ["project1"]}, PipelineContext("run-id")
+        )
+        assert len(os.listdir(tmp_path)) > 0
+        # This closes QueriesExtractor which in turn closes SqlParsingAggregator
+        source.close()
+        assert len(os.listdir(tmp_path)) == 0
diff --git a/metadata-ingestion/tests/integration/business-glossary/custom_ownership_urns.yml b/metadata-ingestion/tests/integration/business-glossary/custom_ownership_urns.yml
new file mode 100644
index 0000000000000..94aae6999a3f5
--- /dev/null
+++ b/metadata-ingestion/tests/integration/business-glossary/custom_ownership_urns.yml
@@ -0,0 +1,38 @@
+version: "1"
+source: DataHub
+owners:
+  users:
+    - mjames
+url: "https://github.com/datahub-project/datahub/"
+
+nodes:
+  - name: Custom URN Types
+    description: Testing custom ownership URN types
+    owners:
+      - type: urn:li:ownershipType:custom_type_1
+        users:
+          - user1
+        groups:
+          - group1
+      - type: urn:li:ownershipType:custom_type_2
+        users:
+          - user2
+    terms:
+      - name: Mixed URN Types
+        description: Term with custom URN types
+        owners:
+          - type: urn:li:ownershipType:custom_type_3
+            users:
+              - user3
+          - type: urn:li:ownershipType:custom_type_4
+            groups:
+              - group2
+      - name: Mixed Standard and URN
+        description: Term with both standard and URN types
+        owners:
+          - type: DEVELOPER
+            users:
+              - dev1
+          - type: urn:li:ownershipType:custom_type_5
+            groups:
+              - group3
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/business-glossary/custom_ownership_urns_golden.json b/metadata-ingestion/tests/integration/business-glossary/custom_ownership_urns_golden.json
new file mode 100644
index 0000000000000..2fc3de77efd8e
--- /dev/null
+++ b/metadata-ingestion/tests/integration/business-glossary/custom_ownership_urns_golden.json
@@ -0,0 +1,188 @@
+[
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": {
+            "urn": "urn:li:glossaryNode:Custom URN Types",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": {
+                        "customProperties": {},
+                        "definition": "Testing custom ownership URN types",
+                        "name": "Custom URN Types"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:user1",
+                                "type": "CUSTOM",
+                                "typeUrn": "urn:li:ownershipType:custom_type_1"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:group1",
+                                "type": "CUSTOM",
+                                "typeUrn": "urn:li:ownershipType:custom_type_1"
+                            },
+                            {
+                                "owner": "urn:li:corpuser:user2",
+                                "type": "CUSTOM",
+                                "typeUrn": "urn:li:ownershipType:custom_type_2"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-dlsmlo",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
+            "urn": "urn:li:glossaryTerm:Custom URN Types.Mixed URN Types",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
+                        "customProperties": {},
+                        "name": "Mixed URN Types",
+                        "definition": "Term with custom URN types",
+                        "parentNode": "urn:li:glossaryNode:Custom URN Types",
+                        "termSource": "INTERNAL",
+                        "sourceRef": "DataHub",
+                        "sourceUrl": "https://github.com/datahub-project/datahub/"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:user3",
+                                "type": "CUSTOM",
+                                "typeUrn": "urn:li:ownershipType:custom_type_3"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:group2",
+                                "type": "CUSTOM",
+                                "typeUrn": "urn:li:ownershipType:custom_type_4"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-dlsmlo",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
+            "urn": "urn:li:glossaryTerm:Custom URN Types.Mixed Standard and URN",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
+                        "customProperties": {},
+                        "name": "Mixed Standard and URN",
+                        "definition": "Term with both standard and URN types",
+                        "parentNode": "urn:li:glossaryNode:Custom URN Types",
+                        "termSource": "INTERNAL",
+                        "sourceRef": "DataHub",
+                        "sourceUrl": "https://github.com/datahub-project/datahub/"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:dev1",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:group3",
+                                "type": "CUSTOM",
+                                "typeUrn": "urn:li:ownershipType:custom_type_5"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-dlsmlo",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryNode",
+    "entityUrn": "urn:li:glossaryNode:Custom URN Types",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-dlsmlo",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryTerm",
+    "entityUrn": "urn:li:glossaryTerm:Custom URN Types.Mixed Standard and URN",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-dlsmlo",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryTerm",
+    "entityUrn": "urn:li:glossaryTerm:Custom URN Types.Mixed URN Types",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-dlsmlo",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/business-glossary/multiple_owners_different_types.yml b/metadata-ingestion/tests/integration/business-glossary/multiple_owners_different_types.yml
new file mode 100644
index 0000000000000..efcc594f758fa
--- /dev/null
+++ b/metadata-ingestion/tests/integration/business-glossary/multiple_owners_different_types.yml
@@ -0,0 +1,39 @@
+version: "1"
+source: DataHub
+owners:
+  users:
+    - mjames
+url: "https://github.com/datahub-project/datahub/"
+
+nodes:
+  - name: Different Owner Types
+    description: Testing multiple owners with different types
+    owners:
+      - type: DEVELOPER
+        users:
+          - dev1
+        groups:
+          - engineering
+      - type: DATAOWNER
+        users:
+          - owner1
+        groups:
+          - data_stewards
+      - type: PRODUCER
+        users:
+          - producer1
+    terms:
+      - name: Mixed Ownership
+        description: Term with different owner types
+        owners:
+          - type: STAKEHOLDER
+            users:
+              - stakeholder1
+            groups:
+              - business
+          - type: DEVELOPER
+            users:
+              - dev2
+          - type: DATAOWNER
+            groups:
+              - compliance
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/business-glossary/multiple_owners_different_types_golden.json b/metadata-ingestion/tests/integration/business-glossary/multiple_owners_different_types_golden.json
new file mode 100644
index 0000000000000..4cec348708291
--- /dev/null
+++ b/metadata-ingestion/tests/integration/business-glossary/multiple_owners_different_types_golden.json
@@ -0,0 +1,138 @@
+[
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": {
+            "urn": "urn:li:glossaryNode:Different Owner Types",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": {
+                        "customProperties": {},
+                        "definition": "Testing multiple owners with different types",
+                        "name": "Different Owner Types"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:dev1",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:engineering",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpuser:owner1",
+                                "type": "DATAOWNER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:data_stewards",
+                                "type": "DATAOWNER"
+                            },
+                            {
+                                "owner": "urn:li:corpuser:producer1",
+                                "type": "PRODUCER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-2te9j9",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
+            "urn": "urn:li:glossaryTerm:Different Owner Types.Mixed Ownership",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
+                        "customProperties": {},
+                        "name": "Mixed Ownership",
+                        "definition": "Term with different owner types",
+                        "parentNode": "urn:li:glossaryNode:Different Owner Types",
+                        "termSource": "INTERNAL",
+                        "sourceRef": "DataHub",
+                        "sourceUrl": "https://github.com/datahub-project/datahub/"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:stakeholder1",
+                                "type": "STAKEHOLDER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:business",
+                                "type": "STAKEHOLDER"
+                            },
+                            {
+                                "owner": "urn:li:corpuser:dev2",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:compliance",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-2te9j9",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryNode",
+    "entityUrn": "urn:li:glossaryNode:Different Owner Types",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-2te9j9",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryTerm",
+    "entityUrn": "urn:li:glossaryTerm:Different Owner Types.Mixed Ownership",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-2te9j9",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/business-glossary/multiple_owners_same_type.yml b/metadata-ingestion/tests/integration/business-glossary/multiple_owners_same_type.yml
new file mode 100644
index 0000000000000..8fb093b8b5899
--- /dev/null
+++ b/metadata-ingestion/tests/integration/business-glossary/multiple_owners_same_type.yml
@@ -0,0 +1,37 @@
+version: "1"
+source: DataHub
+owners:
+  users:
+    - mjames
+url: "https://github.com/datahub-project/datahub/"
+
+nodes:
+  - name: Multiple Owners
+    description: Testing multiple owners with same type
+    owners:
+      - type: DEVELOPER
+        users:
+          - dev1
+          - dev2
+        groups:
+          - engineering
+      - type: DEVELOPER
+        users:
+          - dev3
+        groups:
+          - qa
+    terms:
+      - name: Multiple Dev Owners
+        description: Term owned by multiple developers
+        owners:
+          - type: DEVELOPER
+            users:
+              - dev4
+              - dev5
+            groups:
+              - platform
+          - type: DEVELOPER
+            users:
+              - dev6
+            groups:
+              - infra
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/business-glossary/multiple_owners_same_type_golden.json b/metadata-ingestion/tests/integration/business-glossary/multiple_owners_same_type_golden.json
new file mode 100644
index 0000000000000..9342682510d84
--- /dev/null
+++ b/metadata-ingestion/tests/integration/business-glossary/multiple_owners_same_type_golden.json
@@ -0,0 +1,142 @@
+[
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": {
+            "urn": "urn:li:glossaryNode:Multiple Owners",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": {
+                        "customProperties": {},
+                        "definition": "Testing multiple owners with same type",
+                        "name": "Multiple Owners"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:dev1",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpuser:dev2",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:engineering",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpuser:dev3",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:qa",
+                                "type": "DEVELOPER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-0l66l7",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
+            "urn": "urn:li:glossaryTerm:Multiple Owners.Multiple Dev Owners",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
+                        "customProperties": {},
+                        "name": "Multiple Dev Owners",
+                        "definition": "Term owned by multiple developers",
+                        "parentNode": "urn:li:glossaryNode:Multiple Owners",
+                        "termSource": "INTERNAL",
+                        "sourceRef": "DataHub",
+                        "sourceUrl": "https://github.com/datahub-project/datahub/"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:dev4",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpuser:dev5",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:platform",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpuser:dev6",
+                                "type": "DEVELOPER"
+                            },
+                            {
+                                "owner": "urn:li:corpGroup:infra",
+                                "type": "DEVELOPER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-0l66l7",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryNode",
+    "entityUrn": "urn:li:glossaryNode:Multiple Owners",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-0l66l7",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryTerm",
+    "entityUrn": "urn:li:glossaryTerm:Multiple Owners.Multiple Dev Owners",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-0l66l7",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/business-glossary/single_owner_types.yml b/metadata-ingestion/tests/integration/business-glossary/single_owner_types.yml
new file mode 100644
index 0000000000000..22fc24e6695bc
--- /dev/null
+++ b/metadata-ingestion/tests/integration/business-glossary/single_owner_types.yml
@@ -0,0 +1,39 @@
+version: "1"
+source: DataHub
+url: "https://github.com/datahub-project/datahub/"
+owners:
+  users:
+    - mjames
+
+nodes:
+  - name: Single Owner Types
+    description: Testing different single owner types
+    owners:
+      type: DEVELOPER
+      users:
+        - dev1
+    terms:
+      - name: Developer Owned
+        description: Term owned by developer
+        owners:
+          type: DEVELOPER
+          users:
+            - dev2
+      - name: Data Owner Owned
+        description: Term owned by data owner
+        owners:
+          type: DATAOWNER
+          users:
+            - dataowner1           
+      - name: Producer Owned
+        description: Term owned by producer
+        owners:
+          type: PRODUCER
+          users:
+            - producer1         
+      - name: Stakeholder Owned
+        description: Term owned by stakeholder
+        owners:
+          type: STAKEHOLDER
+          groups:
+            - stakeholders
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/business-glossary/single_owner_types_golden.json b/metadata-ingestion/tests/integration/business-glossary/single_owner_types_golden.json
new file mode 100644
index 0000000000000..006e77f523a10
--- /dev/null
+++ b/metadata-ingestion/tests/integration/business-glossary/single_owner_types_golden.json
@@ -0,0 +1,278 @@
+[
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": {
+            "urn": "urn:li:glossaryNode:Single Owner Types",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": {
+                        "customProperties": {},
+                        "definition": "Testing different single owner types",
+                        "name": "Single Owner Types"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:dev1",
+                                "type": "DEVELOPER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
+            "urn": "urn:li:glossaryTerm:Single Owner Types.Developer Owned",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
+                        "customProperties": {},
+                        "name": "Developer Owned",
+                        "definition": "Term owned by developer",
+                        "parentNode": "urn:li:glossaryNode:Single Owner Types",
+                        "termSource": "INTERNAL",
+                        "sourceRef": "DataHub",
+                        "sourceUrl": "https://github.com/datahub-project/datahub/"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:dev2",
+                                "type": "DEVELOPER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
+            "urn": "urn:li:glossaryTerm:Single Owner Types.Data Owner Owned",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
+                        "customProperties": {},
+                        "name": "Data Owner Owned",
+                        "definition": "Term owned by data owner",
+                        "parentNode": "urn:li:glossaryNode:Single Owner Types",
+                        "termSource": "INTERNAL",
+                        "sourceRef": "DataHub",
+                        "sourceUrl": "https://github.com/datahub-project/datahub/"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:dataowner1",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
+            "urn": "urn:li:glossaryTerm:Single Owner Types.Producer Owned",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
+                        "customProperties": {},
+                        "name": "Producer Owned",
+                        "definition": "Term owned by producer",
+                        "parentNode": "urn:li:glossaryNode:Single Owner Types",
+                        "termSource": "INTERNAL",
+                        "sourceRef": "DataHub",
+                        "sourceUrl": "https://github.com/datahub-project/datahub/"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:producer1",
+                                "type": "PRODUCER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": {
+            "urn": "urn:li:glossaryTerm:Single Owner Types.Stakeholder Owned",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": {
+                        "customProperties": {},
+                        "name": "Stakeholder Owned",
+                        "definition": "Term owned by stakeholder",
+                        "parentNode": "urn:li:glossaryNode:Single Owner Types",
+                        "termSource": "INTERNAL",
+                        "sourceRef": "DataHub",
+                        "sourceUrl": "https://github.com/datahub-project/datahub/"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpGroup:stakeholders",
+                                "type": "STAKEHOLDER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryNode",
+    "entityUrn": "urn:li:glossaryNode:Single Owner Types",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryTerm",
+    "entityUrn": "urn:li:glossaryTerm:Single Owner Types.Data Owner Owned",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryTerm",
+    "entityUrn": "urn:li:glossaryTerm:Single Owner Types.Developer Owned",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryTerm",
+    "entityUrn": "urn:li:glossaryTerm:Single Owner Types.Producer Owned",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "glossaryTerm",
+    "entityUrn": "urn:li:glossaryTerm:Single Owner Types.Stakeholder Owned",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "datahub-business-glossary-2020_04_14-07_00_00-ruwyic",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/business-glossary/test_business_glossary.py b/metadata-ingestion/tests/integration/business-glossary/test_business_glossary.py
index b6e1aca4d4fed..74cf9aa3b528f 100644
--- a/metadata-ingestion/tests/integration/business-glossary/test_business_glossary.py
+++ b/metadata-ingestion/tests/integration/business-glossary/test_business_glossary.py
@@ -3,7 +3,6 @@
 import pytest
 from freezegun import freeze_time
 
-from datahub.ingestion.graph.client import DatahubClientConfig
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.ingestion.source.metadata import business_glossary
 from tests.test_helpers import mce_helpers
@@ -41,7 +40,12 @@ def get_default_recipe(
 @freeze_time(FROZEN_TIME)
 @pytest.mark.integration
 def test_glossary_ingest(
-    mock_datahub_graph, pytestconfig, tmp_path, mock_time, enable_auto_id, golden_file
+    mock_datahub_graph_instance,
+    pytestconfig,
+    tmp_path,
+    mock_time,
+    enable_auto_id,
+    golden_file,
 ):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/business-glossary"
 
@@ -55,9 +59,7 @@ def test_glossary_ingest(
             enable_auto_id=enable_auto_id,
         )
     )
-    pipeline.ctx.graph = mock_datahub_graph(
-        DatahubClientConfig()
-    )  # Mock to resolve domain
+    pipeline.ctx.graph = mock_datahub_graph_instance
     pipeline.run()
     pipeline.raise_from_status()
 
@@ -69,6 +71,134 @@ def test_glossary_ingest(
     )
 
 
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_single_owner_types(
+    mock_datahub_graph_instance,
+    pytestconfig,
+    tmp_path,
+    mock_time,
+):
+    """Test basic single owner cases with different ownership types"""
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/business-glossary"
+    output_mces_path: str = f"{tmp_path}/single_owner_types.json"
+    golden_mces_path: str = f"{test_resources_dir}/single_owner_types_golden.json"
+
+    pipeline = Pipeline.create(
+        get_default_recipe(
+            glossary_yml_file_path=f"{test_resources_dir}/single_owner_types.yml",
+            event_output_file_path=output_mces_path,
+            enable_auto_id=False,
+        )
+    )
+    pipeline.ctx.graph = mock_datahub_graph_instance
+    pipeline.run()
+    pipeline.raise_from_status()
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=output_mces_path,
+        golden_path=golden_mces_path,
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_multiple_owners_same_type(
+    mock_datahub_graph_instance,
+    pytestconfig,
+    tmp_path,
+    mock_time,
+):
+    """Test multiple owners all having the same type"""
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/business-glossary"
+    output_mces_path: str = f"{tmp_path}/multiple_owners_same_type.json"
+    golden_mces_path: str = (
+        f"{test_resources_dir}/multiple_owners_same_type_golden.json"
+    )
+
+    pipeline = Pipeline.create(
+        get_default_recipe(
+            glossary_yml_file_path=f"{test_resources_dir}/multiple_owners_same_type.yml",
+            event_output_file_path=output_mces_path,
+            enable_auto_id=False,
+        )
+    )
+    pipeline.ctx.graph = mock_datahub_graph_instance
+    pipeline.run()
+    pipeline.raise_from_status()
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=output_mces_path,
+        golden_path=golden_mces_path,
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_multiple_owners_different_types(
+    mock_datahub_graph_instance,
+    pytestconfig,
+    tmp_path,
+    mock_time,
+):
+    """Test multiple owners with different types"""
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/business-glossary"
+    output_mces_path: str = f"{tmp_path}/multiple_owners_different_types.json"
+    golden_mces_path: str = (
+        f"{test_resources_dir}/multiple_owners_different_types_golden.json"
+    )
+
+    pipeline = Pipeline.create(
+        get_default_recipe(
+            glossary_yml_file_path=f"{test_resources_dir}/multiple_owners_different_types.yml",
+            event_output_file_path=output_mces_path,
+            enable_auto_id=False,
+        )
+    )
+    pipeline.ctx.graph = mock_datahub_graph_instance
+    pipeline.run()
+    pipeline.raise_from_status()
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=output_mces_path,
+        golden_path=golden_mces_path,
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_custom_ownership_urns(
+    mock_datahub_graph_instance,
+    pytestconfig,
+    tmp_path,
+    mock_time,
+):
+    """Test custom ownership URNs"""
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/business-glossary"
+    output_mces_path: str = f"{tmp_path}/custom_ownership_urns.json"
+    golden_mces_path: str = f"{test_resources_dir}/custom_ownership_urns_golden.json"
+
+    pipeline = Pipeline.create(
+        get_default_recipe(
+            glossary_yml_file_path=f"{test_resources_dir}/custom_ownership_urns.yml",
+            event_output_file_path=output_mces_path,
+            enable_auto_id=False,
+        )
+    )
+    pipeline.ctx.graph = mock_datahub_graph_instance
+    pipeline.run()
+    pipeline.raise_from_status()
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=output_mces_path,
+        golden_path=golden_mces_path,
+    )
+
+
 @freeze_time(FROZEN_TIME)
 def test_auto_id_creation_on_reserved_char():
     id_: str = business_glossary.create_id(["pii", "secure % password"], None, False)
diff --git a/metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json b/metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json
new file mode 100644
index 0000000000000..1823a218ada2e
--- /dev/null
+++ b/metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json
@@ -0,0 +1,2706 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "cassandra",
+                "env": "PROD",
+                "keyspace": "cass_test_1",
+                "durable_writes": "True",
+                "replication": "{\"class\": \"org.apache.cassandra.locator.SimpleStrategy\", \"replication_factor\": \"1\"}"
+            },
+            "name": "cass_test_1",
+            "qualifiedName": "cass_test_1",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516869,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:cassandra"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924399,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924399,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Keyspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924400,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924400,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "information",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"details\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"last_updated\", \"type\": \"timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"person_id\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "details",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "last_updated",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.TimeType": {}
+                        }
+                    },
+                    "nativeDataType": "timestamp",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "person_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "int",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019538,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924405,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924405,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "information",
+            "qualifiedName": "cass_test_1.information",
+            "description": "",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019540,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "all_data_types",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"ascii_column\", \"type\": \"ascii\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"bigint_column\", \"type\": \"bigint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"blob_column\", \"type\": \"blob\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"boolean_column\", \"type\": \"boolean\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"date_column\", \"type\": \"date\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"decimal_column\", \"type\": \"decimal\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"double_column\", \"type\": \"double\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"float_column\", \"type\": \"float\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"frozen_list_column\", \"type\": \"frozen<list<text>>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"frozen_map_column\", \"type\": \"frozen<map<text, text>>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"frozen_set_column\", \"type\": \"frozen<set<text>>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"id\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"inet_column\", \"type\": \"inet\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"int_column\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"list_column\", \"type\": \"list<text>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"map_column\", \"type\": \"map<text, text>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"set_column\", \"type\": \"set<text>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"smallint_column\", \"type\": \"smallint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"text_column\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"time_column\", \"type\": \"time\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"timestamp_column\", \"type\": \"timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"timeuuid_column\", \"type\": \"timeuuid\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"tinyint_column\", \"type\": \"tinyint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"tuple_column\", \"type\": \"frozen<tuple<int, text>>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"uuid_column\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"varchar_column\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"varint_column\", \"type\": \"varint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "ascii_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "ascii",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "bigint_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "blob_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.BytesType": {}
+                        }
+                    },
+                    "nativeDataType": "blob",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "boolean_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.BooleanType": {}
+                        }
+                    },
+                    "nativeDataType": "boolean",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "date_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.DateType": {}
+                        }
+                    },
+                    "nativeDataType": "date",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "decimal_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "decimal",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "double_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "float_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "frozen_list_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NullType": {}
+                        }
+                    },
+                    "nativeDataType": "frozen<list<text>>",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "frozen_map_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NullType": {}
+                        }
+                    },
+                    "nativeDataType": "frozen<map<text, text>>",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "frozen_set_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NullType": {}
+                        }
+                    },
+                    "nativeDataType": "frozen<set<text>>",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "uuid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "inet_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "inet",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "int_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "int",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "list_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NullType": {}
+                        }
+                    },
+                    "nativeDataType": "list<text>",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "map_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NullType": {}
+                        }
+                    },
+                    "nativeDataType": "map<text, text>",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "set_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NullType": {}
+                        }
+                    },
+                    "nativeDataType": "set<text>",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "smallint_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "smallint",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "text_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "time_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.TimeType": {}
+                        }
+                    },
+                    "nativeDataType": "time",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "timestamp_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.TimeType": {}
+                        }
+                    },
+                    "nativeDataType": "timestamp",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "timeuuid_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "timeuuid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "tinyint_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "tinyint",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "tuple_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NullType": {}
+                        }
+                    },
+                    "nativeDataType": "frozen<tuple<int, text>>",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "uuid_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "uuid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "varchar_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "varint_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "varint",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019435,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097192,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "people",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"cass_test_1\", \"table_name\": \"people\", \"column_name\": \"email\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"people\", \"column_name\": \"name\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"people\", \"column_name\": \"person_id\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "email",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "person_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "int",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019563,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924412,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924412,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "people",
+            "qualifiedName": "cass_test_1.people",
+            "description": "",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019564,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
+                    "urn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924406,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924406,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "cassandra",
+                "env": "PROD",
+                "keyspace": "cass_test_2",
+                "durable_writes": "True",
+                "replication": "{\"class\": \"org.apache.cassandra.locator.SimpleStrategy\", \"replication_factor\": \"1\"}"
+            },
+            "name": "cass_test_2",
+            "qualifiedName": "cass_test_2",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516849,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
+                    "urn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924413,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924420,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924413,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924421,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "tasks",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"cass_test_2\", \"table_name\": \"tasks\", \"column_name\": \"details\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_2\", \"table_name\": \"tasks\", \"column_name\": \"last_updated\", \"type\": \"timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_2\", \"table_name\": \"tasks\", \"column_name\": \"status\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_2\", \"table_name\": \"tasks\", \"column_name\": \"task_id\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "details",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "last_updated",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.TimeType": {}
+                        }
+                    },
+                    "nativeDataType": "timestamp",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "status",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "task_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "int",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019516,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924426,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924426,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "tasks",
+            "qualifiedName": "cass_test_2.tasks",
+            "description": "",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019518,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Keyspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924421,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:cassandra"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924420,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
+                    "urn": "urn:li:container:0381892d0717b54887d087eaafd95d2b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924427,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0381892d0717b54887d087eaafd95d2b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731309924427,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:305f73c676989511c67d97ace119138c"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097193,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097193,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "task_status",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"cass_test_2\", \"table_name\": \"task_status\", \"column_name\": \"status\", \"type\": \"text\", \"clustering_order\": \"asc\", \"kind\": \"clustering\", \"position\": 0}, {\"keyspace_name\": \"cass_test_2\", \"table_name\": \"task_status\", \"column_name\": \"task_id\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "status",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "task_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "int",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019525,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:305f73c676989511c67d97ace119138c",
+                    "urn": "urn:li:container:305f73c676989511c67d97ace119138c"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097193,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "all_data_types",
+            "qualifiedName": "example_keyspace.all_data_types",
+            "description": "Table containing all supported Cassandra data types, excluding counters",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019440,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097158,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097161,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "counter_table",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"counter_table\", \"column_name\": \"counter_column\", \"type\": \"counter\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"counter_table\", \"column_name\": \"id\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "counter_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "counter",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "uuid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019446,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "include_all_columns": "False",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "task_status",
+            "qualifiedName": "cass_test_2.task_status",
+            "description": "",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019524,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097198,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": true,
+            "viewLogic": "status IS NOT NULL AND task_id IS NOT NULL",
+            "viewLanguage": "CQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097161,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0381892d0717b54887d087eaafd95d2b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097163,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
+                    "urn": "urn:li:container:0381892d0717b54887d087eaafd95d2b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097163,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:305f73c676989511c67d97ace119138c"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097199,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097198,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD),status)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD),status)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD),task_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD),task_id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731447296444,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:305f73c676989511c67d97ace119138c",
+                    "urn": "urn:li:container:305f73c676989511c67d97ace119138c"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097199,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "counter_table",
+            "qualifiedName": "example_keyspace.counter_table",
+            "description": "Separate table containing only counter column",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019447,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:cassandra"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097186,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "cassandra",
+                "env": "PROD",
+                "keyspace": "example_keyspace",
+                "durable_writes": "True",
+                "replication": "{\"class\": \"org.apache.cassandra.locator.SimpleStrategy\", \"replication_factor\": \"1\"}"
+            },
+            "name": "example_keyspace",
+            "qualifiedName": "example_keyspace",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516801,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097185,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Keyspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097186,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310097186,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "shopping_cart",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"item_count\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"last_update_timestamp\", \"type\": \"timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"userid\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "item_count",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "int",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "last_update_timestamp",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.TimeType": {}
+                        }
+                    },
+                    "nativeDataType": "timestamp",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "userid",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "text",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019453,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),ascii_column)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD),ascii_column)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),bigint_column)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD),bigint_column)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731447296557,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731410842611,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731410842610,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": true,
+            "viewLogic": "id IS NOT NULL AND ascii_column IS NOT NULL",
+            "viewLanguage": "CQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310103458,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "shopping_cart",
+            "qualifiedName": "example_keyspace.shopping_cart",
+            "description": "",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019455,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310103456,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:305f73c676989511c67d97ace119138c",
+                    "urn": "urn:li:container:305f73c676989511c67d97ace119138c"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731410842612,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:305f73c676989511c67d97ace119138c"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731410842611,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310103457,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "example_view_1",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_1\", \"column_name\": \"ascii_column\", \"type\": \"ascii\", \"clustering_order\": \"asc\", \"kind\": \"clustering\", \"position\": 0}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_1\", \"column_name\": \"bigint_column\", \"type\": \"bigint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_1\", \"column_name\": \"id\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "ascii_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "ascii",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "bigint_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "uuid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019464,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:305f73c676989511c67d97ace119138c"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310103460,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:305f73c676989511c67d97ace119138c",
+                    "urn": "urn:li:container:305f73c676989511c67d97ace119138c"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310103461,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "include_all_columns": "False",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "example_view_1",
+            "qualifiedName": "example_keyspace.example_view_1",
+            "description": "Example view definition with id and ascii_column",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019464,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:305f73c676989511c67d97ace119138c",
+                    "urn": "urn:li:container:305f73c676989511c67d97ace119138c"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310942175,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310942171,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310942172,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": true,
+            "viewLogic": "id IS NOT NULL AND ascii_column IS NOT NULL",
+            "viewLanguage": "CQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310942172,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),ascii_column)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD),ascii_column)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),float_column)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD),float_column)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731447296594,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "bloom_filter_fp_chance": "0.01",
+                "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}",
+                "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}",
+                "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}",
+                "crc_check_chance": "1.0",
+                "include_all_columns": "False",
+                "dclocal_read_repair_chance": "0.0",
+                "default_time_to_live": "0",
+                "extensions": "{}",
+                "gc_grace_seconds": "864000",
+                "max_index_interval": "2048",
+                "min_index_interval": "128",
+                "memtable_flush_period_in_ms": "0",
+                "read_repair_chance": "0.0",
+                "speculative_retry": "99p"
+            },
+            "name": "example_view_2",
+            "qualifiedName": "example_keyspace.example_view_2",
+            "description": "Example view definition with id and ascii_column",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019474,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "example_view_2",
+            "platform": "urn:li:dataPlatform:cassandra",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_2\", \"column_name\": \"ascii_column\", \"type\": \"ascii\", \"clustering_order\": \"asc\", \"kind\": \"clustering\", \"position\": 0}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_2\", \"column_name\": \"float_column\", \"type\": \"float\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_2\", \"column_name\": \"id\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]"
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "ascii_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "ascii",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "float_column",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "uuid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731591019474,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:305f73c676989511c67d97ace119138c"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731310942175,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1731579516915,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 27,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "ascii_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "bigint_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "blob_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "boolean_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "date_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "decimal_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "double_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "float_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "frozen_list_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "frozen_map_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "frozen_set_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "id",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "inet_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "int_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "list_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "map_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "set_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "smallint_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "text_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "time_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "timestamp_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "timeuuid_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "tinyint_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "tuple_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "uuid_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "varchar_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "varint_column",
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516925,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1731579516959,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 3,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "email",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "name",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "person_id",
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516960,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1731579516904,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 2,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "counter_column",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "id",
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516915,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1731579516939,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 4,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "details",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "last_updated",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "status",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "task_id",
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516950,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1731579516950,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 3,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "details",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "last_updated",
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "person_id",
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516959,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1731579516925,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 9,
+            "columnCount": 3,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "item_count",
+                    "uniqueCount": 5,
+                    "nullCount": 4,
+                    "min": "2",
+                    "max": "100",
+                    "mean": "46.4",
+                    "median": "50.0",
+                    "stdev": "38.44",
+                    "sampleValues": [
+                        "5",
+                        "100",
+                        "75",
+                        "2",
+                        "50"
+                    ]
+                },
+                {
+                    "fieldPath": "last_update_timestamp",
+                    "uniqueCount": 9,
+                    "nullCount": 0,
+                    "min": "2024-11-01 00:00:00",
+                    "max": "2024-11-09 00:00:00",
+                    "sampleValues": [
+                        "2024-11-08 00:00:00",
+                        "2024-11-06 00:00:00",
+                        "2024-11-02 00:00:00",
+                        "2024-11-03 00:00:00",
+                        "2024-11-05 00:00:00"
+                    ]
+                },
+                {
+                    "fieldPath": "userid",
+                    "uniqueCount": 9,
+                    "nullCount": 0,
+                    "min": "1234",
+                    "max": "9876",
+                    "sampleValues": [
+                        "1240",
+                        "1238",
+                        "1234",
+                        "1235",
+                        "1237"
+                    ]
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1731579516939,
+        "runId": "cassandra-test",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/cassandra/docker-compose.yml b/metadata-ingestion/tests/integration/cassandra/docker-compose.yml
new file mode 100644
index 0000000000000..a1a2a3b97d134
--- /dev/null
+++ b/metadata-ingestion/tests/integration/cassandra/docker-compose.yml
@@ -0,0 +1,38 @@
+version: "1"
+services:
+  test-cassandra:
+    image: cassandra:latest
+    container_name: test-cassandra
+    ports:
+      - 9042:9042
+    volumes:
+      - ./setup/cassandra.yaml:/etc/cassandra/cassandra.yaml
+      - ./setup/init_keyspaces.cql:/docker-entrypoint-initdb.d/init_keyspaces.cql
+    networks:
+      - testnet
+    healthcheck:
+      test: ["CMD-SHELL", "cqlsh -e 'describe keyspaces' || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 10
+
+  test-cassandra-load-keyspace:
+    container_name: test-cassandra-load-keyspace
+    image: cassandra:latest
+    depends_on:
+      test-cassandra:
+        condition: service_healthy
+    volumes:
+      - ./setup/init_keyspaces.cql:/init_keyspaces.cql
+    command: /bin/bash -c "echo loading cassandra keyspace && cqlsh test-cassandra -f init_keyspaces.cql"
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+        max_attempts: 3
+        window: 100s
+    networks:
+      - testnet
+
+networks:
+  testnet:
diff --git a/metadata-ingestion/tests/integration/cassandra/setup/cassandra.yaml b/metadata-ingestion/tests/integration/cassandra/setup/cassandra.yaml
new file mode 100644
index 0000000000000..2dee71a0b5d1b
--- /dev/null
+++ b/metadata-ingestion/tests/integration/cassandra/setup/cassandra.yaml
@@ -0,0 +1,1827 @@
+# ---------------------------------------------------------------------------------------- #
+# NOTE:
+# This yaml has been copied from the official Cassandra Docker image:
+# (https://github.com/datastax/docker-images/blob/master/config-templates/DSE/6.0.0/cassandra.yaml)
+# 
+# Key interests are in setting the AllowAllNetworkAuthorizer,
+# and in enabling materialized view feature.
+# ---------------------------------------------------------------------------------------- #
+
+# Cassandra storage config YAML
+
+# NOTE:
+#   See https://cassandra.apache.org/doc/latest/configuration/ for
+#   full explanations of configuration directives
+# /NOTE
+
+# The name of the cluster. This is mainly used to prevent machines in
+# one logical cluster from joining another.
+cluster_name: 'Test Cluster'
+
+# This defines the number of tokens randomly assigned to this node on the ring
+# The more tokens, relative to other nodes, the larger the proportion of data
+# that this node will store. You probably want all nodes to have the same number
+# of tokens assuming they have equal hardware capability.
+#
+# If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility,
+# and will use the initial_token as described below.
+#
+# Specifying initial_token will override this setting on the node's initial start,
+# on subsequent starts, this setting will apply even if initial token is set.
+#
+# See https://cassandra.apache.org/doc/latest/getting_started/production.html#tokens for
+# best practice information about num_tokens.
+#
+num_tokens: 16
+
+# Triggers automatic allocation of num_tokens tokens for this node. The allocation
+# algorithm attempts to choose tokens in a way that optimizes replicated load over
+# the nodes in the datacenter for the replica factor.
+#
+# The load assigned to each node will be close to proportional to its number of
+# vnodes.
+#
+# Only supported with the Murmur3Partitioner.
+
+# Replica factor is determined via the replication strategy used by the specified
+# keyspace.
+# allocate_tokens_for_keyspace: KEYSPACE
+
+# Replica factor is explicitly set, regardless of keyspace or datacenter.
+# This is the replica factor within the datacenter, like NTS.
+allocate_tokens_for_local_replication_factor: 3
+
+# initial_token allows you to specify tokens manually.  While you can use it with
+# vnodes (num_tokens > 1, above) -- in which case you should provide a 
+# comma-separated list -- it's primarily used when adding nodes to legacy clusters 
+# that do not have vnodes enabled.
+# initial_token:
+
+# May either be "true" or "false" to enable globally
+hinted_handoff_enabled: true
+
+# When hinted_handoff_enabled is true, a black list of data centers that will not
+# perform hinted handoff
+# hinted_handoff_disabled_datacenters:
+#    - DC1
+#    - DC2
+
+# this defines the maximum amount of time a dead host will have hints
+# generated.  After it has been dead this long, new hints for it will not be
+# created until it has been seen alive and gone down again.
+# Min unit: ms
+max_hint_window: 3h
+
+# Maximum throttle in KiBs per second, per delivery thread.  This will be
+# reduced proportionally to the number of nodes in the cluster.  (If there
+# are two nodes in the cluster, each delivery thread will use the maximum
+# rate; if there are three, each will throttle to half of the maximum,
+# since we expect two nodes to be delivering hints simultaneously.)
+# Min unit: KiB
+hinted_handoff_throttle: 1024KiB
+
+# Number of threads with which to deliver hints;
+# Consider increasing this number when you have multi-dc deployments, since
+# cross-dc handoff tends to be slower
+max_hints_delivery_threads: 2
+
+# Directory where Cassandra should store hints.
+# If not set, the default directory is $CASSANDRA_HOME/data/hints.
+# hints_directory: /var/lib/cassandra/hints
+
+# How often hints should be flushed from the internal buffers to disk.
+# Will *not* trigger fsync.
+# Min unit: ms
+hints_flush_period: 10000ms
+
+# Maximum size for a single hints file, in mebibytes.
+# Min unit: MiB
+max_hints_file_size: 128MiB
+
+# The file size limit to store hints for an unreachable host, in mebibytes.
+# Once the local hints files have reached the limit, no more new hints will be created.
+# Set a non-positive value will disable the size limit.
+# max_hints_size_per_host: 0MiB
+
+# Enable / disable automatic cleanup for the expired and orphaned hints file.
+# Disable the option in order to preserve those hints on the disk.
+auto_hints_cleanup_enabled: false
+
+# Compression to apply to the hint files. If omitted, hints files
+# will be written uncompressed. LZ4, Snappy, and Deflate compressors
+# are supported.
+#hints_compression:
+#   - class_name: LZ4Compressor
+#     parameters:
+#         -
+
+# Enable / disable persistent hint windows.
+#
+# If set to false, a hint will be stored only in case a respective node
+# that hint is for is down less than or equal to max_hint_window.
+#
+# If set to true, a hint will be stored in case there is not any
+# hint which was stored earlier than max_hint_window. This is for cases
+# when a node keeps to restart and hints are not delivered yet, we would be saving
+# hints for that node indefinitely.
+#
+# Defaults to true.
+#
+# hint_window_persistent_enabled: true
+
+# Maximum throttle in KiBs per second, total. This will be
+# reduced proportionally to the number of nodes in the cluster.
+# Min unit: KiB
+batchlog_replay_throttle: 1024KiB
+
+# Authentication backend, implementing IAuthenticator; used to identify users
+# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator,
+# PasswordAuthenticator}.
+#
+# - AllowAllAuthenticator performs no checks - set it to disable authentication.
+# - PasswordAuthenticator relies on username/password pairs to authenticate
+#   users. It keeps usernames and hashed passwords in system_auth.roles table.
+#   Please increase system_auth keyspace replication factor if you use this authenticator.
+#   If using PasswordAuthenticator, CassandraRoleManager must also be used (see below)
+authenticator: AllowAllAuthenticator
+
+# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions
+# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer,
+# CassandraAuthorizer}.
+#
+# - AllowAllAuthorizer allows any action to any user - set it to disable authorization.
+# - CassandraAuthorizer stores permissions in system_auth.role_permissions table. Please
+#   increase system_auth keyspace replication factor if you use this authorizer.
+authorizer: AllowAllAuthorizer
+
+# Part of the Authentication & Authorization backend, implementing IRoleManager; used
+# to maintain grants and memberships between roles.
+# Out of the box, Cassandra provides org.apache.cassandra.auth.CassandraRoleManager,
+# which stores role information in the system_auth keyspace. Most functions of the
+# IRoleManager require an authenticated login, so unless the configured IAuthenticator
+# actually implements authentication, most of this functionality will be unavailable.
+#
+# - CassandraRoleManager stores role data in the system_auth keyspace. Please
+#   increase system_auth keyspace replication factor if you use this role manager.
+role_manager: CassandraRoleManager
+
+# Network authorization backend, implementing INetworkAuthorizer; used to restrict user
+# access to certain DCs
+# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllNetworkAuthorizer,
+# CassandraNetworkAuthorizer}.
+#
+# - AllowAllNetworkAuthorizer allows access to any DC to any user - set it to disable authorization.
+# - CassandraNetworkAuthorizer stores permissions in system_auth.network_permissions table. Please
+#   increase system_auth keyspace replication factor if you use this authorizer.
+network_authorizer: AllowAllNetworkAuthorizer
+
+# Depending on the auth strategy of the cluster, it can be beneficial to iterate
+# from root to table (root -> ks -> table) instead of table to root (table -> ks -> root).
+# As the auth entries are whitelisting, once a permission is found you know it to be
+# valid. We default to false as the legacy behavior is to query at the table level then
+# move back up to the root. See CASSANDRA-17016 for details.
+# traverse_auth_from_root: false
+
+# Validity period for roles cache (fetching granted roles can be an expensive
+# operation depending on the role manager, CassandraRoleManager is one example)
+# Granted roles are cached for authenticated sessions in AuthenticatedUser and
+# after the period specified here, become eligible for (async) reload.
+# Defaults to 2000, set to 0 to disable caching entirely.
+# Will be disabled automatically for AllowAllAuthenticator.
+# For a long-running cache using roles_cache_active_update, consider
+# setting to something longer such as a daily validation: 86400000
+# Min unit: ms
+roles_validity: 2000ms
+
+# Refresh interval for roles cache (if enabled).
+# After this interval, cache entries become eligible for refresh. Upon next
+# access, an async reload is scheduled and the old value returned until it
+# completes. If roles_validity is non-zero, then this must be
+# also.
+# This setting is also used to inform the interval of auto-updating if
+# using roles_cache_active_update.
+# Defaults to the same value as roles_validity.
+# For a long-running cache, consider setting this to 60000 (1 hour) etc.
+# Min unit: ms
+# roles_update_interval: 2000ms
+
+# If true, cache contents are actively updated by a background task at the
+# interval set by roles_update_interval. If false, cache entries
+# become eligible for refresh after their update interval. Upon next access,
+# an async reload is scheduled and the old value returned until it completes.
+# roles_cache_active_update: false
+
+# Validity period for permissions cache (fetching permissions can be an
+# expensive operation depending on the authorizer, CassandraAuthorizer is
+# one example). Defaults to 2000, set to 0 to disable.
+# Will be disabled automatically for AllowAllAuthorizer.
+# For a long-running cache using permissions_cache_active_update, consider
+# setting to something longer such as a daily validation: 86400000ms
+# Min unit: ms
+permissions_validity: 2000ms
+
+# Refresh interval for permissions cache (if enabled).
+# After this interval, cache entries become eligible for refresh. Upon next
+# access, an async reload is scheduled and the old value returned until it
+# completes. If permissions_validity is non-zero, then this must be
+# also.
+# This setting is also used to inform the interval of auto-updating if
+# using permissions_cache_active_update.
+# Defaults to the same value as permissions_validity.
+# For a longer-running permissions cache, consider setting to update hourly (60000)
+# Min unit: ms
+# permissions_update_interval: 2000ms
+
+# If true, cache contents are actively updated by a background task at the
+# interval set by permissions_update_interval. If false, cache entries
+# become eligible for refresh after their update interval. Upon next access,
+# an async reload is scheduled and the old value returned until it completes.
+# permissions_cache_active_update: false
+
+# Validity period for credentials cache. This cache is tightly coupled to
+# the provided PasswordAuthenticator implementation of IAuthenticator. If
+# another IAuthenticator implementation is configured, this cache will not
+# be automatically used and so the following settings will have no effect.
+# Please note, credentials are cached in their encrypted form, so while
+# activating this cache may reduce the number of queries made to the
+# underlying table, it may not  bring a significant reduction in the
+# latency of individual authentication attempts.
+# Defaults to 2000, set to 0 to disable credentials caching.
+# For a long-running cache using credentials_cache_active_update, consider
+# setting to something longer such as a daily validation: 86400000
+# Min unit: ms
+credentials_validity: 2000ms
+
+# Refresh interval for credentials cache (if enabled).
+# After this interval, cache entries become eligible for refresh. Upon next
+# access, an async reload is scheduled and the old value returned until it
+# completes. If credentials_validity is non-zero, then this must be
+# also.
+# This setting is also used to inform the interval of auto-updating if
+# using credentials_cache_active_update.
+# Defaults to the same value as credentials_validity.
+# For a longer-running permissions cache, consider setting to update hourly (60000)
+# Min unit: ms
+# credentials_update_interval: 2000ms
+
+# If true, cache contents are actively updated by a background task at the
+# interval set by credentials_update_interval. If false (default), cache entries
+# become eligible for refresh after their update interval. Upon next access,
+# an async reload is scheduled and the old value returned until it completes.
+# credentials_cache_active_update: false
+
+# The partitioner is responsible for distributing groups of rows (by
+# partition key) across nodes in the cluster. The partitioner can NOT be
+# changed without reloading all data.  If you are adding nodes or upgrading,
+# you should set this to the same partitioner that you are currently using.
+#
+# The default partitioner is the Murmur3Partitioner. Older partitioners
+# such as the RandomPartitioner, ByteOrderedPartitioner, and
+# OrderPreservingPartitioner have been included for backward compatibility only.
+# For new clusters, you should NOT change this value.
+#
+partitioner: org.apache.cassandra.dht.Murmur3Partitioner
+
+# Directories where Cassandra should store data on disk. If multiple
+# directories are specified, Cassandra will spread data evenly across 
+# them by partitioning the token ranges.
+# If not set, the default directory is $CASSANDRA_HOME/data/data.
+# data_file_directories:
+#     - /var/lib/cassandra/data
+
+# Directory were Cassandra should store the data of the local system keyspaces.
+# By default Cassandra will store the data of the local system keyspaces in the first of the data directories specified
+# by data_file_directories.
+# This approach ensures that if one of the other disks is lost Cassandra can continue to operate. For extra security
+# this setting allows to store those data on a different directory that provides redundancy.
+# local_system_data_file_directory:
+
+# commit log.  when running on magnetic HDD, this should be a
+# separate spindle than the data directories.
+# If not set, the default directory is $CASSANDRA_HOME/data/commitlog.
+# commitlog_directory: /var/lib/cassandra/commitlog
+
+# Enable / disable CDC functionality on a per-node basis. This modifies the logic used
+# for write path allocation rejection (standard: never reject. cdc: reject Mutation
+# containing a CDC-enabled table if at space limit in cdc_raw_directory).
+cdc_enabled: false
+
+# CommitLogSegments are moved to this directory on flush if cdc_enabled: true and the
+# segment contains mutations for a CDC-enabled table. This should be placed on a
+# separate spindle than the data directories. If not set, the default directory is
+# $CASSANDRA_HOME/data/cdc_raw.
+# cdc_raw_directory: /var/lib/cassandra/cdc_raw
+
+# Policy for data disk failures:
+#
+# die
+#   shut down gossip and client transports and kill the JVM for any fs errors or
+#   single-sstable errors, so the node can be replaced.
+#
+# stop_paranoid
+#   shut down gossip and client transports even for single-sstable errors,
+#   kill the JVM for errors during startup.
+#
+# stop
+#   shut down gossip and client transports, leaving the node effectively dead, but
+#   can still be inspected via JMX, kill the JVM for errors during startup.
+#
+# best_effort
+#    stop using the failed disk and respond to requests based on
+#    remaining available sstables.  This means you WILL see obsolete
+#    data at CL.ONE!
+#
+# ignore
+#    ignore fatal errors and let requests fail, as in pre-1.2 Cassandra
+disk_failure_policy: stop
+
+# Policy for commit disk failures:
+#
+# die
+#   shut down the node and kill the JVM, so the node can be replaced.
+#
+# stop
+#   shut down the node, leaving the node effectively dead, but
+#   can still be inspected via JMX.
+#
+# stop_commit
+#   shutdown the commit log, letting writes collect but
+#   continuing to service reads, as in pre-2.0.5 Cassandra
+#
+# ignore
+#   ignore fatal errors and let the batches fail
+commit_failure_policy: stop
+
+# Maximum size of the native protocol prepared statement cache
+#
+# Valid values are either "auto" (omitting the value) or a value greater 0.
+#
+# Note that specifying a too large value will result in long running GCs and possbily
+# out-of-memory errors. Keep the value at a small fraction of the heap.
+#
+# If you constantly see "prepared statements discarded in the last minute because
+# cache limit reached" messages, the first step is to investigate the root cause
+# of these messages and check whether prepared statements are used correctly -
+# i.e. use bind markers for variable parts.
+#
+# Do only change the default value, if you really have more prepared statements than
+# fit in the cache. In most cases it is not neccessary to change this value.
+# Constantly re-preparing statements is a performance penalty.
+#
+# Default value ("auto") is 1/256th of the heap or 10MiB, whichever is greater
+# Min unit: MiB
+prepared_statements_cache_size:
+
+# Maximum size of the key cache in memory.
+#
+# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the
+# minimum, sometimes more. The key cache is fairly tiny for the amount of
+# time it saves, so it's worthwhile to use it at large numbers.
+# The row cache saves even more time, but must contain the entire row,
+# so it is extremely space-intensive. It's best to only use the
+# row cache if you have hot rows or static rows.
+#
+# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup.
+#
+# Default value is empty to make it "auto" (min(5% of Heap (in MiB), 100MiB)). Set to 0 to disable key cache.
+# Min unit: MiB
+key_cache_size:
+
+# Duration in seconds after which Cassandra should
+# save the key cache. Caches are saved to saved_caches_directory as
+# specified in this configuration file.
+#
+# Saved caches greatly improve cold-start speeds, and is relatively cheap in
+# terms of I/O for the key cache. Row cache saving is much more expensive and
+# has limited use.
+#
+# Default is 14400 or 4 hours.
+# Min unit: s
+key_cache_save_period: 4h
+
+# Number of keys from the key cache to save
+# Disabled by default, meaning all keys are going to be saved
+# key_cache_keys_to_save: 100
+
+# Row cache implementation class name. Available implementations:
+#
+# org.apache.cassandra.cache.OHCProvider
+#   Fully off-heap row cache implementation (default).
+#
+# org.apache.cassandra.cache.SerializingCacheProvider
+#   This is the row cache implementation availabile
+#   in previous releases of Cassandra.
+# row_cache_class_name: org.apache.cassandra.cache.OHCProvider
+
+# Maximum size of the row cache in memory.
+# Please note that OHC cache implementation requires some additional off-heap memory to manage
+# the map structures and some in-flight memory during operations before/after cache entries can be
+# accounted against the cache capacity. This overhead is usually small compared to the whole capacity.
+# Do not specify more memory that the system can afford in the worst usual situation and leave some
+# headroom for OS block level cache. Do never allow your system to swap.
+#
+# Default value is 0, to disable row caching.
+# Min unit: MiB
+row_cache_size: 0MiB
+
+# Duration in seconds after which Cassandra should save the row cache.
+# Caches are saved to saved_caches_directory as specified in this configuration file.
+#
+# Saved caches greatly improve cold-start speeds, and is relatively cheap in
+# terms of I/O for the key cache. Row cache saving is much more expensive and
+# has limited use.
+#
+# Default is 0 to disable saving the row cache.
+# Min unit: s
+row_cache_save_period: 0s
+
+# Number of keys from the row cache to save.
+# Specify 0 (which is the default), meaning all keys are going to be saved
+# row_cache_keys_to_save: 100
+
+# Maximum size of the counter cache in memory.
+#
+# Counter cache helps to reduce counter locks' contention for hot counter cells.
+# In case of RF = 1 a counter cache hit will cause Cassandra to skip the read before
+# write entirely. With RF > 1 a counter cache hit will still help to reduce the duration
+# of the lock hold, helping with hot counter cell updates, but will not allow skipping
+# the read entirely. Only the local (clock, count) tuple of a counter cell is kept
+# in memory, not the whole counter, so it's relatively cheap.
+#
+# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup.
+#
+# Default value is empty to make it "auto" (min(2.5% of Heap (in MiB), 50MiB)). Set to 0 to disable counter cache.
+# NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache.
+# Min unit: MiB
+counter_cache_size:
+
+# Duration in seconds after which Cassandra should
+# save the counter cache (keys only). Caches are saved to saved_caches_directory as
+# specified in this configuration file.
+#
+# Default is 7200 or 2 hours.
+# Min unit: s
+counter_cache_save_period: 7200s
+
+# Number of keys from the counter cache to save
+# Disabled by default, meaning all keys are going to be saved
+# counter_cache_keys_to_save: 100
+
+# saved caches
+# If not set, the default directory is $CASSANDRA_HOME/data/saved_caches.
+# saved_caches_directory: /var/lib/cassandra/saved_caches
+
+# Number of seconds the server will wait for each cache (row, key, etc ...) to load while starting
+# the Cassandra process. Setting this to zero is equivalent to disabling all cache loading on startup
+# while still having the cache during runtime.
+# Min unit: s
+# cache_load_timeout: 30s
+
+# commitlog_sync may be either "periodic", "group", or "batch." 
+# 
+# When in batch mode, Cassandra won't ack writes until the commit log
+# has been flushed to disk.  Each incoming write will trigger the flush task.
+# commitlog_sync_batch_window_in_ms is a deprecated value. Previously it had
+# almost no value, and is being removed.
+#
+# commitlog_sync_batch_window_in_ms: 2
+#
+# group mode is similar to batch mode, where Cassandra will not ack writes
+# until the commit log has been flushed to disk. The difference is group
+# mode will wait up to commitlog_sync_group_window between flushes.
+#
+# Min unit: ms
+# commitlog_sync_group_window: 1000ms
+#
+# the default option is "periodic" where writes may be acked immediately
+# and the CommitLog is simply synced every commitlog_sync_period
+# milliseconds.
+commitlog_sync: periodic
+# Min unit: ms
+commitlog_sync_period: 10000ms
+
+# When in periodic commitlog mode, the number of milliseconds to block writes
+# while waiting for a slow disk flush to complete.
+# Min unit: ms
+# periodic_commitlog_sync_lag_block:
+
+# The size of the individual commitlog file segments.  A commitlog
+# segment may be archived, deleted, or recycled once all the data
+# in it (potentially from each columnfamily in the system) has been
+# flushed to sstables.
+#
+# The default size is 32, which is almost always fine, but if you are
+# archiving commitlog segments (see commitlog_archiving.properties),
+# then you probably want a finer granularity of archiving; 8 or 16 MB
+# is reasonable.
+# Max mutation size is also configurable via max_mutation_size setting in
+# cassandra.yaml. The default is half the size commitlog_segment_size in bytes.
+# This should be positive and less than 2048.
+#
+# NOTE: If max_mutation_size is set explicitly then commitlog_segment_size must
+# be set to at least twice the size of max_mutation_size
+#
+# Min unit: MiB
+commitlog_segment_size: 32MiB
+
+# Compression to apply to the commit log. If omitted, the commit log
+# will be written uncompressed.  LZ4, Snappy, and Deflate compressors
+# are supported.
+# commitlog_compression:
+#   - class_name: LZ4Compressor
+#     parameters:
+#         -
+
+# Compression to apply to SSTables as they flush for compressed tables.
+# Note that tables without compression enabled do not respect this flag.
+#
+# As high ratio compressors like LZ4HC, Zstd, and Deflate can potentially
+# block flushes for too long, the default is to flush with a known fast
+# compressor in those cases. Options are:
+#
+# none : Flush without compressing blocks but while still doing checksums.
+# fast : Flush with a fast compressor. If the table is already using a
+#        fast compressor that compressor is used.
+# table: Always flush with the same compressor that the table uses. This
+#        was the pre 4.0 behavior.
+#
+# flush_compression: fast
+
+# any class that implements the SeedProvider interface and has a
+# constructor that takes a Map<String, String> of parameters will do.
+seed_provider:
+  # Addresses of hosts that are deemed contact points.
+  # Cassandra nodes use this list of hosts to find each other and learn
+  # the topology of the ring.  You must change this if you are running
+  # multiple nodes!
+  - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+    parameters:
+      # seeds is actually a comma-delimited list of addresses.
+      # Ex: "<ip1>,<ip2>,<ip3>"
+      - seeds: "172.18.0.2"
+
+# For workloads with more data than can fit in memory, Cassandra's
+# bottleneck will be reads that need to fetch data from
+# disk. "concurrent_reads" should be set to (16 * number_of_drives) in
+# order to allow the operations to enqueue low enough in the stack
+# that the OS and drives can reorder them. Same applies to
+# "concurrent_counter_writes", since counter writes read the current
+# values before incrementing and writing them back.
+#
+# On the other hand, since writes are almost never IO bound, the ideal
+# number of "concurrent_writes" is dependent on the number of cores in
+# your system; (8 * number_of_cores) is a good rule of thumb.
+concurrent_reads: 32
+concurrent_writes: 32
+concurrent_counter_writes: 32
+
+# For materialized view writes, as there is a read involved, so this should
+# be limited by the less of concurrent reads or concurrent writes.
+concurrent_materialized_view_writes: 32
+
+# Maximum memory to use for inter-node and client-server networking buffers.
+#
+# Defaults to the smaller of 1/16 of heap or 128MB. This pool is allocated off-heap,
+# so is in addition to the memory allocated for heap. The cache also has on-heap
+# overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size
+# if the default 64k chunk size is used).
+# Memory is only allocated when needed.
+# Min unit: MiB
+# networking_cache_size: 128MiB
+
+# Enable the sstable chunk cache.  The chunk cache will store recently accessed
+# sections of the sstable in-memory as uncompressed buffers.
+# file_cache_enabled: false
+
+# Maximum memory to use for sstable chunk cache and buffer pooling.
+# 32MB of this are reserved for pooling buffers, the rest is used for chunk cache
+# that holds uncompressed sstable chunks.
+# Defaults to the smaller of 1/4 of heap or 512MB. This pool is allocated off-heap,
+# so is in addition to the memory allocated for heap. The cache also has on-heap
+# overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size
+# if the default 64k chunk size is used).
+# Memory is only allocated when needed.
+# Min unit: MiB
+# file_cache_size: 512MiB
+
+# Flag indicating whether to allocate on or off heap when the sstable buffer
+# pool is exhausted, that is when it has exceeded the maximum memory
+# file_cache_size, beyond which it will not cache buffers but allocate on request.
+
+# buffer_pool_use_heap_if_exhausted: true
+
+# The strategy for optimizing disk read
+# Possible values are:
+# ssd (for solid state disks, the default)
+# spinning (for spinning disks)
+# disk_optimization_strategy: ssd
+
+# Total permitted memory to use for memtables. Cassandra will stop
+# accepting writes when the limit is exceeded until a flush completes,
+# and will trigger a flush based on memtable_cleanup_threshold
+# If omitted, Cassandra will set both to 1/4 the size of the heap.
+# Min unit: MiB
+# memtable_heap_space: 2048MiB
+# Min unit: MiB
+# memtable_offheap_space: 2048MiB
+
+# memtable_cleanup_threshold is deprecated. The default calculation
+# is the only reasonable choice. See the comments on  memtable_flush_writers
+# for more information.
+#
+# Ratio of occupied non-flushing memtable size to total permitted size
+# that will trigger a flush of the largest memtable. Larger mct will
+# mean larger flushes and hence less compaction, but also less concurrent
+# flush activity which can make it difficult to keep your disks fed
+# under heavy write load.
+#
+# memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1)
+# memtable_cleanup_threshold: 0.11
+
+# Specify the way Cassandra allocates and manages memtable memory.
+# Options are:
+#
+# heap_buffers
+#   on heap nio buffers
+#
+# offheap_buffers
+#   off heap (direct) nio buffers
+#
+# offheap_objects
+#    off heap objects
+memtable_allocation_type: heap_buffers
+
+# Limit memory usage for Merkle tree calculations during repairs. The default
+# is 1/16th of the available heap. The main tradeoff is that smaller trees
+# have less resolution, which can lead to over-streaming data. If you see heap
+# pressure during repairs, consider lowering this, but you cannot go below
+# one mebibyte. If you see lots of over-streaming, consider raising
+# this or using subrange repair.
+#
+# For more details see https://issues.apache.org/jira/browse/CASSANDRA-14096.
+#
+# Min unit: MiB
+# repair_session_space:
+
+# Total space to use for commit logs on disk.
+#
+# If space gets above this value, Cassandra will flush every dirty CF
+# in the oldest segment and remove it.  So a small total commitlog space
+# will tend to cause more flush activity on less-active columnfamilies.
+#
+# The default value is the smaller of 8192, and 1/4 of the total space
+# of the commitlog volume.
+#
+# commitlog_total_space: 8192MiB
+
+# This sets the number of memtable flush writer threads per disk
+# as well as the total number of memtables that can be flushed concurrently.
+# These are generally a combination of compute and IO bound.
+#
+# Memtable flushing is more CPU efficient than memtable ingest and a single thread
+# can keep up with the ingest rate of a whole server on a single fast disk
+# until it temporarily becomes IO bound under contention typically with compaction.
+# At that point you need multiple flush threads. At some point in the future
+# it may become CPU bound all the time.
+#
+# You can tell if flushing is falling behind using the MemtablePool.BlockedOnAllocation
+# metric which should be 0, but will be non-zero if threads are blocked waiting on flushing
+# to free memory.
+#
+# memtable_flush_writers defaults to two for a single data directory.
+# This means that two  memtables can be flushed concurrently to the single data directory.
+# If you have multiple data directories the default is one memtable flushing at a time
+# but the flush will use a thread per data directory so you will get two or more writers.
+#
+# Two is generally enough to flush on a fast disk [array] mounted as a single data directory.
+# Adding more flush writers will result in smaller more frequent flushes that introduce more
+# compaction overhead.
+#
+# There is a direct tradeoff between number of memtables that can be flushed concurrently
+# and flush size and frequency. More is not better you just need enough flush writers
+# to never stall waiting for flushing to free memory.
+#
+# memtable_flush_writers: 2
+
+# Total space to use for change-data-capture logs on disk.
+#
+# If space gets above this value, Cassandra will throw WriteTimeoutException
+# on Mutations including tables with CDC enabled. A CDCCompactor is responsible
+# for parsing the raw CDC logs and deleting them when parsing is completed.
+#
+# The default value is the min of 4096 MiB and 1/8th of the total space
+# of the drive where cdc_raw_directory resides.
+# Min unit: MiB
+# cdc_total_space: 4096MiB
+
+# When we hit our cdc_raw limit and the CDCCompactor is either running behind
+# or experiencing backpressure, we check at the following interval to see if any
+# new space for cdc-tracked tables has been made available. Default to 250ms
+# Min unit: ms
+# cdc_free_space_check_interval: 250ms
+
+# A fixed memory pool size in MB for for SSTable index summaries. If left
+# empty, this will default to 5% of the heap size. If the memory usage of
+# all index summaries exceeds this limit, SSTables with low read rates will
+# shrink their index summaries in order to meet this limit.  However, this
+# is a best-effort process. In extreme conditions Cassandra may need to use
+# more than this amount of memory.
+# Min unit: KiB
+index_summary_capacity:
+
+# How frequently index summaries should be resampled.  This is done
+# periodically to redistribute memory from the fixed-size pool to sstables
+# proportional their recent read rates.  Setting to null value will disable this
+# process, leaving existing index summaries at their current sampling level.
+# Min unit: m
+index_summary_resize_interval: 60m
+
+# Whether to, when doing sequential writing, fsync() at intervals in
+# order to force the operating system to flush the dirty
+# buffers. Enable this to avoid sudden dirty buffer flushing from
+# impacting read latencies. Almost always a good idea on SSDs; not
+# necessarily on platters.
+trickle_fsync: false
+# Min unit: KiB
+trickle_fsync_interval: 10240KiB
+
+# TCP port, for commands and data
+# For security reasons, you should not expose this port to the internet.  Firewall it if needed.
+storage_port: 7000
+
+# SSL port, for legacy encrypted communication. This property is unused unless enabled in
+# server_encryption_options (see below). As of cassandra 4.0, this property is deprecated
+# as a single port can be used for either/both secure and insecure connections.
+# For security reasons, you should not expose this port to the internet. Firewall it if needed.
+ssl_storage_port: 7001
+
+# Address or interface to bind to and tell other Cassandra nodes to connect to.
+# You _must_ change this if you want multiple nodes to be able to communicate!
+#
+# Set listen_address OR listen_interface, not both.
+#
+# Leaving it blank leaves it up to InetAddress.getLocalHost(). This
+# will always do the Right Thing _if_ the node is properly configured
+# (hostname, name resolution, etc), and the Right Thing is to use the
+# address associated with the hostname (it might not be). If unresolvable
+# it will fall back to InetAddress.getLoopbackAddress(), which is wrong for production systems.
+#
+# Setting listen_address to 0.0.0.0 is always wrong.
+#
+listen_address: 172.18.0.2
+
+# Set listen_address OR listen_interface, not both. Interfaces must correspond
+# to a single address, IP aliasing is not supported.
+# listen_interface: eth0
+
+# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address
+# you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4
+# address will be used. If true the first ipv6 address will be used. Defaults to false preferring
+# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6.
+# listen_interface_prefer_ipv6: false
+
+# Address to broadcast to other Cassandra nodes
+# Leaving this blank will set it to the same value as listen_address
+broadcast_address: 172.18.0.2
+
+# When using multiple physical network interfaces, set this
+# to true to listen on broadcast_address in addition to
+# the listen_address, allowing nodes to communicate in both
+# interfaces.
+# Ignore this property if the network configuration automatically
+# routes  between the public and private networks such as EC2.
+# listen_on_broadcast_address: false
+
+# Internode authentication backend, implementing IInternodeAuthenticator;
+# used to allow/disallow connections from peer nodes.
+# internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator
+
+# Whether to start the native transport server.
+# The address on which the native transport is bound is defined by rpc_address.
+start_native_transport: true
+# port for the CQL native transport to listen for clients on
+# For security reasons, you should not expose this port to the internet.  Firewall it if needed.
+native_transport_port: 9042
+# Enabling native transport encryption in client_encryption_options allows you to either use
+# encryption for the standard port or to use a dedicated, additional port along with the unencrypted
+# standard native_transport_port.
+# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption
+# for native_transport_port. Setting native_transport_port_ssl to a different value
+# from native_transport_port will use encryption for native_transport_port_ssl while
+# keeping native_transport_port unencrypted.
+# native_transport_port_ssl: 9142
+# The maximum threads for handling requests (note that idle threads are stopped
+# after 30 seconds so there is not corresponding minimum setting).
+# native_transport_max_threads: 128
+#
+# The maximum size of allowed frame. Frame (requests) larger than this will
+# be rejected as invalid. The default is 16MiB. If you're changing this parameter,
+# you may want to adjust max_value_size accordingly. This should be positive and less than 2048.
+# Min unit: MiB
+# native_transport_max_frame_size: 16MiB
+
+# The maximum number of concurrent client connections.
+# The default is -1, which means unlimited.
+# native_transport_max_concurrent_connections: -1
+
+# The maximum number of concurrent client connections per source ip.
+# The default is -1, which means unlimited.
+# native_transport_max_concurrent_connections_per_ip: -1
+
+# Controls whether Cassandra honors older, yet currently supported, protocol versions.
+# The default is true, which means all supported protocols will be honored.
+native_transport_allow_older_protocols: true
+
+# Controls when idle client connections are closed. Idle connections are ones that had neither reads
+# nor writes for a time period.
+#
+# Clients may implement heartbeats by sending OPTIONS native protocol message after a timeout, which
+# will reset idle timeout timer on the server side. To close idle client connections, corresponding
+# values for heartbeat intervals have to be set on the client side.
+#
+# Idle connection timeouts are disabled by default.
+# Min unit: ms
+# native_transport_idle_timeout: 60000ms
+
+# When enabled, limits the number of native transport requests dispatched for processing per second.
+# Behavior once the limit has been breached depends on the value of THROW_ON_OVERLOAD specified in
+# the STARTUP message sent by the client during connection establishment. (See section "4.1.1. STARTUP"
+# in "CQL BINARY PROTOCOL v5".) With the THROW_ON_OVERLOAD flag enabled, messages that breach the limit
+# are dropped, and an OverloadedException is thrown for the client to handle. When the flag is not
+# enabled, the server will stop consuming messages from the channel/socket, putting backpressure on
+# the client while already dispatched messages are processed.
+# native_transport_rate_limiting_enabled: false
+# native_transport_max_requests_per_second: 1000000
+
+# The address or interface to bind the native transport server to.
+#
+# Set rpc_address OR rpc_interface, not both.
+#
+# Leaving rpc_address blank has the same effect as on listen_address
+# (i.e. it will be based on the configured hostname of the node).
+#
+# Note that unlike listen_address, you can specify 0.0.0.0, but you must also
+# set broadcast_rpc_address to a value other than 0.0.0.0.
+#
+# For security reasons, you should not expose this port to the internet.  Firewall it if needed.
+rpc_address: 0.0.0.0
+
+# Set rpc_address OR rpc_interface, not both. Interfaces must correspond
+# to a single address, IP aliasing is not supported.
+# rpc_interface: eth1
+
+# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address
+# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4
+# address will be used. If true the first ipv6 address will be used. Defaults to false preferring
+# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6.
+# rpc_interface_prefer_ipv6: false
+
+# RPC address to broadcast to drivers and other Cassandra nodes. This cannot
+# be set to 0.0.0.0. If left blank, this will be set to the value of
+# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must
+# be set.
+broadcast_rpc_address: 172.18.0.2
+
+# enable or disable keepalive on rpc/native connections
+rpc_keepalive: true
+
+# Uncomment to set socket buffer size for internode communication
+# Note that when setting this, the buffer size is limited by net.core.wmem_max
+# and when not setting it it is defined by net.ipv4.tcp_wmem
+# See also:
+# /proc/sys/net/core/wmem_max
+# /proc/sys/net/core/rmem_max
+# /proc/sys/net/ipv4/tcp_wmem
+# /proc/sys/net/ipv4/tcp_wmem
+# and 'man tcp'
+# Min unit: B
+# internode_socket_send_buffer_size:
+
+# Uncomment to set socket buffer size for internode communication
+# Note that when setting this, the buffer size is limited by net.core.wmem_max
+# and when not setting it it is defined by net.ipv4.tcp_wmem
+# Min unit: B
+# internode_socket_receive_buffer_size:
+
+# Set to true to have Cassandra create a hard link to each sstable
+# flushed or streamed locally in a backups/ subdirectory of the
+# keyspace data.  Removing these links is the operator's
+# responsibility.
+incremental_backups: false
+
+# Whether or not to take a snapshot before each compaction.  Be
+# careful using this option, since Cassandra won't clean up the
+# snapshots for you.  Mostly useful if you're paranoid when there
+# is a data format change.
+snapshot_before_compaction: false
+
+# Whether or not a snapshot is taken of the data before keyspace truncation
+# or dropping of column families. The STRONGLY advised default of true 
+# should be used to provide data safety. If you set this flag to false, you will
+# lose data on truncation or drop.
+auto_snapshot: true
+
+# Adds a time-to-live (TTL) to auto snapshots generated by table
+# truncation or drop (when enabled).
+# After the TTL is elapsed, the snapshot is automatically cleared.
+# By default, auto snapshots *do not* have TTL, uncomment the property below
+# to enable TTL on auto snapshots.
+# Accepted units: d (days), h (hours) or m (minutes)
+# auto_snapshot_ttl: 30d
+
+# The act of creating or clearing a snapshot involves creating or removing
+# potentially tens of thousands of links, which can cause significant performance
+# impact, especially on consumer grade SSDs. A non-zero value here can
+# be used to throttle these links to avoid negative performance impact of
+# taking and clearing snapshots
+snapshot_links_per_second: 0
+
+# Granularity of the collation index of rows within a partition.
+# Increase if your rows are large, or if you have a very large
+# number of rows per partition.  The competing goals are these:
+#
+# - a smaller granularity means more index entries are generated
+#   and looking up rows withing the partition by collation column
+#   is faster
+# - but, Cassandra will keep the collation index in memory for hot
+#   rows (as part of the key cache), so a larger granularity means
+#   you can cache more hot rows
+# Min unit: KiB
+column_index_size: 64KiB
+
+# Per sstable indexed key cache entries (the collation index in memory
+# mentioned above) exceeding this size will not be held on heap.
+# This means that only partition information is held on heap and the
+# index entries are read from disk.
+#
+# Note that this size refers to the size of the
+# serialized index information and not the size of the partition.
+# Min unit: KiB
+column_index_cache_size: 2KiB
+
+# Number of simultaneous compactions to allow, NOT including
+# validation "compactions" for anti-entropy repair.  Simultaneous
+# compactions can help preserve read performance in a mixed read/write
+# workload, by mitigating the tendency of small sstables to accumulate
+# during a single long running compactions. The default is usually
+# fine and if you experience problems with compaction running too
+# slowly or too fast, you should look at
+# compaction_throughput first.
+#
+# concurrent_compactors defaults to the smaller of (number of disks,
+# number of cores), with a minimum of 2 and a maximum of 8.
+# 
+# If your data directories are backed by SSD, you should increase this
+# to the number of cores.
+# concurrent_compactors: 1
+
+# Number of simultaneous repair validations to allow. If not set or set to
+# a value less than 1, it defaults to the value of concurrent_compactors.
+# To set a value greeater than concurrent_compactors at startup, the system
+# property cassandra.allow_unlimited_concurrent_validations must be set to
+# true. To dynamically resize to a value > concurrent_compactors on a running
+# node, first call the bypassConcurrentValidatorsLimit method on the
+# org.apache.cassandra.db:type=StorageService mbean
+# concurrent_validations: 0
+
+# Number of simultaneous materialized view builder tasks to allow.
+concurrent_materialized_view_builders: 1
+
+# Throttles compaction to the given total throughput across the entire
+# system. The faster you insert data, the faster you need to compact in
+# order to keep the sstable count down, but in general, setting this to
+# 16 to 32 times the rate you are inserting data is more than sufficient.
+# Setting this to 0 disables throttling. Note that this accounts for all types
+# of compaction, including validation compaction (building Merkle trees
+# for repairs).
+compaction_throughput: 64MiB/s
+
+# When compacting, the replacement sstable(s) can be opened before they
+# are completely written, and used in place of the prior sstables for
+# any range that has been written. This helps to smoothly transfer reads 
+# between the sstables, reducing page cache churn and keeping hot rows hot
+# Set sstable_preemptive_open_interval to null for disabled which is equivalent to
+# sstable_preemptive_open_interval_in_mb being negative
+# Min unit: MiB
+sstable_preemptive_open_interval: 50MiB
+
+# Starting from 4.1 sstables support UUID based generation identifiers. They are disabled by default
+# because once enabled, there is no easy way to downgrade. When the node is restarted with this option
+# set to true, each newly created sstable will have a UUID based generation identifier and such files are
+# not readable by previous Cassandra versions. At some point, this option will become true by default
+# and eventually get removed from the configuration.
+uuid_sstable_identifiers_enabled: false
+
+# When enabled, permits Cassandra to zero-copy stream entire eligible
+# SSTables between nodes, including every component.
+# This speeds up the network transfer significantly subject to
+# throttling specified by entire_sstable_stream_throughput_outbound,
+# and entire_sstable_inter_dc_stream_throughput_outbound
+# for inter-DC transfers.
+# Enabling this will reduce the GC pressure on sending and receiving node.
+# When unset, the default is enabled. While this feature tries to keep the
+# disks balanced, it cannot guarantee it. This feature will be automatically
+# disabled if internode encryption is enabled.
+# stream_entire_sstables: true
+
+# Throttles entire SSTable outbound streaming file transfers on
+# this node to the given total throughput in Mbps.
+# Setting this value to 0 it disables throttling.
+# When unset, the default is 200 Mbps or 24 MiB/s.
+# entire_sstable_stream_throughput_outbound: 24MiB/s
+
+# Throttles entire SSTable file streaming between datacenters.
+# Setting this value to 0 disables throttling for entire SSTable inter-DC file streaming.
+# When unset, the default is 200 Mbps or 24 MiB/s.
+# entire_sstable_inter_dc_stream_throughput_outbound: 24MiB/s
+
+# Throttles all outbound streaming file transfers on this node to the
+# given total throughput in Mbps. This is necessary because Cassandra does
+# mostly sequential IO when streaming data during bootstrap or repair, which
+# can lead to saturating the network connection and degrading rpc performance.
+# When unset, the default is 200 Mbps or 24 MiB/s.
+# stream_throughput_outbound: 24MiB/s
+
+# Throttles all streaming file transfer between the datacenters,
+# this setting allows users to throttle inter dc stream throughput in addition
+# to throttling all network stream traffic as configured with
+# stream_throughput_outbound_megabits_per_sec
+# When unset, the default is 200 Mbps or 24 MiB/s.
+# inter_dc_stream_throughput_outbound: 24MiB/s
+
+# Server side timeouts for requests. The server will return a timeout exception
+# to the client if it can't complete an operation within the corresponding
+# timeout. Those settings are a protection against:
+#   1) having client wait on an operation that might never terminate due to some
+#      failures.
+#   2) operations that use too much CPU/read too much data (leading to memory build
+#      up) by putting a limit to how long an operation will execute.
+# For this reason, you should avoid putting these settings too high. In other words,
+# if you are timing out requests because of underlying resource constraints then
+# increasing the timeout will just cause more problems. Of course putting them too
+# low is equally ill-advised since clients could get timeouts even for successful
+# operations just because the timeout setting is too tight.
+
+# How long the coordinator should wait for read operations to complete.
+# Lowest acceptable value is 10 ms.
+# Min unit: ms
+read_request_timeout: 5000ms
+# How long the coordinator should wait for seq or index scans to complete.
+# Lowest acceptable value is 10 ms.
+# Min unit: ms
+range_request_timeout: 10000ms
+# How long the coordinator should wait for writes to complete.
+# Lowest acceptable value is 10 ms.
+# Min unit: ms
+write_request_timeout: 2000ms
+# How long the coordinator should wait for counter writes to complete.
+# Lowest acceptable value is 10 ms.
+# Min unit: ms
+counter_write_request_timeout: 5000ms
+# How long a coordinator should continue to retry a CAS operation
+# that contends with other proposals for the same row.
+# Lowest acceptable value is 10 ms.
+# Min unit: ms
+cas_contention_timeout: 1000ms
+# How long the coordinator should wait for truncates to complete
+# (This can be much longer, because unless auto_snapshot is disabled
+# we need to flush first so we can snapshot before removing the data.)
+# Lowest acceptable value is 10 ms.
+# Min unit: ms
+truncate_request_timeout: 60000ms
+# The default timeout for other, miscellaneous operations.
+# Lowest acceptable value is 10 ms.
+# Min unit: ms
+request_timeout: 10000ms
+
+# Defensive settings for protecting Cassandra from true network partitions.
+# See (CASSANDRA-14358) for details.
+#
+# The amount of time to wait for internode tcp connections to establish.
+# Min unit: ms
+# internode_tcp_connect_timeout: 2000ms
+#
+# The amount of time unacknowledged data is allowed on a connection before we throw out the connection
+# Note this is only supported on Linux + epoll, and it appears to behave oddly above a setting of 30000
+# (it takes much longer than 30s) as of Linux 4.12. If you want something that high set this to 0
+# which picks up the OS default and configure the net.ipv4.tcp_retries2 sysctl to be ~8.
+# Min unit: ms
+# internode_tcp_user_timeout: 30000ms
+
+# The amount of time unacknowledged data is allowed on a streaming connection.
+# The default is 5 minutes. Increase it or set it to 0 in order to increase the timeout.
+# Min unit: ms
+# internode_streaming_tcp_user_timeout: 300000ms
+
+# Global, per-endpoint and per-connection limits imposed on messages queued for delivery to other nodes
+# and waiting to be processed on arrival from other nodes in the cluster.  These limits are applied to the on-wire
+# size of the message being sent or received.
+#
+# The basic per-link limit is consumed in isolation before any endpoint or global limit is imposed.
+# Each node-pair has three links: urgent, small and large.  So any given node may have a maximum of
+# N*3*(internode_application_send_queue_capacity+internode_application_receive_queue_capacity)
+# messages queued without any coordination between them although in practice, with token-aware routing, only RF*tokens
+# nodes should need to communicate with significant bandwidth.
+#
+# The per-endpoint limit is imposed on all messages exceeding the per-link limit, simultaneously with the global limit,
+# on all links to or from a single node in the cluster.
+# The global limit is imposed on all messages exceeding the per-link limit, simultaneously with the per-endpoint limit,
+# on all links to or from any node in the cluster.
+#
+# Min unit: B
+# internode_application_send_queue_capacity: 4MiB
+# internode_application_send_queue_reserve_endpoint_capacity: 128MiB
+# internode_application_send_queue_reserve_global_capacity: 512MiB
+# internode_application_receive_queue_capacity: 4MiB
+# internode_application_receive_queue_reserve_endpoint_capacity: 128MiB
+# internode_application_receive_queue_reserve_global_capacity: 512MiB
+
+
+# How long before a node logs slow queries. Select queries that take longer than
+# this timeout to execute, will generate an aggregated log message, so that slow queries
+# can be identified. Set this value to zero to disable slow query logging.
+# Min unit: ms
+slow_query_log_timeout: 500ms
+
+# Enable operation timeout information exchange between nodes to accurately
+# measure request timeouts.  If disabled, replicas will assume that requests
+# were forwarded to them instantly by the coordinator, which means that
+# under overload conditions we will waste that much extra time processing 
+# already-timed-out requests.
+#
+# Warning: It is generally assumed that users have setup NTP on their clusters, and that clocks are modestly in sync, 
+# since this is a requirement for general correctness of last write wins.
+# internode_timeout: true
+
+# Set period for idle state control messages for earlier detection of failed streams
+# This node will send a keep-alive message periodically on the streaming's control channel.
+# This ensures that any eventual SocketTimeoutException will occur within 2 keep-alive cycles
+# If the node cannot send, or timeouts sending, the keep-alive message on the netty control channel
+# the stream session is closed.
+# Default value is 300s (5 minutes), which means stalled streams
+# are detected within 10 minutes
+# Specify 0 to disable.
+# Min unit: s
+# streaming_keep_alive_period: 300s
+
+# Limit number of connections per host for streaming
+# Increase this when you notice that joins are CPU-bound rather that network
+# bound (for example a few nodes with big files).
+# streaming_connections_per_host: 1
+
+# Settings for stream stats tracking; used by system_views.streaming table
+# How long before a stream is evicted from tracking; this impacts both historic and currently running
+# streams.
+# streaming_state_expires: 3d
+# How much memory may be used for tracking before evicting session from tracking; once crossed
+# historic and currently running streams maybe impacted.
+# streaming_state_size: 40MiB
+# Enable/Disable tracking of streaming stats
+# streaming_stats_enabled: true
+
+# Allows denying configurable access (rw/rr) to operations on configured ks, table, and partitions, intended for use by
+# operators to manage cluster health vs application access. See CASSANDRA-12106 and CEP-13 for more details.
+# partition_denylist_enabled: false
+
+# denylist_writes_enabled: true
+# denylist_reads_enabled: true
+# denylist_range_reads_enabled: true
+
+# The interval at which keys in the cache for denylisting will "expire" and async refresh from the backing DB.
+# Note: this serves only as a fail-safe, as the usage pattern is expected to be "mutate state, refresh cache" on any
+# changes to the underlying denylist entries. See documentation for details.
+# Min unit: s
+# denylist_refresh: 600s
+
+# In the event of errors on attempting to load the denylist cache, retry on this interval.
+# Min unit: s
+# denylist_initial_load_retry: 5s
+
+# We cap the number of denylisted keys allowed per table to keep things from growing unbounded. Nodes will warn above
+# this limit while allowing new denylisted keys to be inserted. Denied keys are loaded in natural query / clustering
+# ordering by partition key in case of overflow.
+# denylist_max_keys_per_table: 1000
+
+# We cap the total number of denylisted keys allowed in the cluster to keep things from growing unbounded.
+# Nodes will warn on initial cache load that there are too many keys and be direct the operator to trim down excess
+# entries to within the configured limits.
+# denylist_max_keys_total: 10000
+
+# Since the denylist in many ways serves to protect the health of the cluster from partitions operators have identified
+# as being in a bad state, we usually want more robustness than just CL.ONE on operations to/from these tables to
+# ensure that these safeguards are in place. That said, we allow users to configure this if they're so inclined.
+# denylist_consistency_level: QUORUM
+
+# phi value that must be reached for a host to be marked down.
+# most users should never need to adjust this.
+# phi_convict_threshold: 8
+
+# endpoint_snitch -- Set this to a class that implements
+# IEndpointSnitch.  The snitch has two functions:
+#
+# - it teaches Cassandra enough about your network topology to route
+#   requests efficiently
+# - it allows Cassandra to spread replicas around your cluster to avoid
+#   correlated failures. It does this by grouping machines into
+#   "datacenters" and "racks."  Cassandra will do its best not to have
+#   more than one replica on the same "rack" (which may not actually
+#   be a physical location)
+#
+# CASSANDRA WILL NOT ALLOW YOU TO SWITCH TO AN INCOMPATIBLE SNITCH
+# ONCE DATA IS INSERTED INTO THE CLUSTER.  This would cause data loss.
+# This means that if you start with the default SimpleSnitch, which
+# locates every node on "rack1" in "datacenter1", your only options
+# if you need to add another datacenter are GossipingPropertyFileSnitch
+# (and the older PFS).  From there, if you want to migrate to an
+# incompatible snitch like Ec2Snitch you can do it by adding new nodes
+# under Ec2Snitch (which will locate them in a new "datacenter") and
+# decommissioning the old ones.
+#
+# Out of the box, Cassandra provides:
+#
+# SimpleSnitch:
+#    Treats Strategy order as proximity. This can improve cache
+#    locality when disabling read repair.  Only appropriate for
+#    single-datacenter deployments.
+#
+# GossipingPropertyFileSnitch
+#    This should be your go-to snitch for production use.  The rack
+#    and datacenter for the local node are defined in
+#    cassandra-rackdc.properties and propagated to other nodes via
+#    gossip.  If cassandra-topology.properties exists, it is used as a
+#    fallback, allowing migration from the PropertyFileSnitch.
+#
+# PropertyFileSnitch:
+#    Proximity is determined by rack and data center, which are
+#    explicitly configured in cassandra-topology.properties.
+#
+# Ec2Snitch:
+#    Appropriate for EC2 deployments in a single Region. Loads Region
+#    and Availability Zone information from the EC2 API. The Region is
+#    treated as the datacenter, and the Availability Zone as the rack.
+#    Only private IPs are used, so this will not work across multiple
+#    Regions.
+#
+# Ec2MultiRegionSnitch:
+#    Uses public IPs as broadcast_address to allow cross-region
+#    connectivity.  (Thus, you should set seed addresses to the public
+#    IP as well.) You will need to open the storage_port or
+#    ssl_storage_port on the public IP firewall.  (For intra-Region
+#    traffic, Cassandra will switch to the private IP after
+#    establishing a connection.)
+#
+# RackInferringSnitch:
+#    Proximity is determined by rack and data center, which are
+#    assumed to correspond to the 3rd and 2nd octet of each node's IP
+#    address, respectively.  Unless this happens to match your
+#    deployment conventions, this is best used as an example of
+#    writing a custom Snitch class and is provided in that spirit.
+#
+# You can use a custom Snitch by setting this to the full class name
+# of the snitch, which will be assumed to be on your classpath.
+endpoint_snitch: SimpleSnitch
+
+# controls how often to perform the more expensive part of host score
+# calculation
+# Min unit: ms
+dynamic_snitch_update_interval: 100ms
+# controls how often to reset all host scores, allowing a bad host to
+# possibly recover
+# Min unit: ms
+dynamic_snitch_reset_interval: 600000ms
+# if set greater than zero, this will allow
+# 'pinning' of replicas to hosts in order to increase cache capacity.
+# The badness threshold will control how much worse the pinned host has to be
+# before the dynamic snitch will prefer other replicas over it.  This is
+# expressed as a double which represents a percentage.  Thus, a value of
+# 0.2 means Cassandra would continue to prefer the static snitch values
+# until the pinned host was 20% worse than the fastest.
+dynamic_snitch_badness_threshold: 1.0
+
+# Configure server-to-server internode encryption
+#
+# JVM and netty defaults for supported SSL socket protocols and cipher suites can
+# be replaced using custom encryption options. This is not recommended
+# unless you have policies in place that dictate certain settings, or
+# need to disable vulnerable ciphers or protocols in case the JVM cannot
+# be updated.
+#
+# FIPS compliant settings can be configured at JVM level and should not
+# involve changing encryption settings here:
+# https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/FIPS.html
+#
+# **NOTE** this default configuration is an insecure configuration. If you need to
+# enable server-to-server encryption generate server keystores (and truststores for mutual
+# authentication) per:
+# http://download.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
+# Then perform the following configuration changes:
+#
+# Step 1: Set internode_encryption=<dc|rack|all> and explicitly set optional=true. Restart all nodes
+#
+# Step 2: Set optional=false (or remove it) and if you generated truststores and want to use mutual
+# auth set require_client_auth=true. Restart all nodes
+server_encryption_options:
+  # On outbound connections, determine which type of peers to securely connect to.
+  #   The available options are :
+  #     none : Do not encrypt outgoing connections
+  #     dc   : Encrypt connections to peers in other datacenters but not within datacenters
+  #     rack : Encrypt connections to peers in other racks but not within racks
+  #     all  : Always use encrypted connections
+  internode_encryption: none
+  # When set to true, encrypted and unencrypted connections are allowed on the storage_port
+  # This should _only be true_ while in unencrypted or transitional operation
+  # optional defaults to true if internode_encryption is none
+  # optional: true
+  # If enabled, will open up an encrypted listening socket on ssl_storage_port. Should only be used
+  # during upgrade to 4.0; otherwise, set to false.
+  legacy_ssl_storage_port_enabled: false
+  # Set to a valid keystore if internode_encryption is dc, rack or all
+  keystore: conf/.keystore
+  keystore_password: cassandra
+  # Configure the way Cassandra creates SSL contexts.
+  # To use PEM-based key material, see org.apache.cassandra.security.PEMBasedSslContextFactory
+  # ssl_context_factory:
+  #     # Must be an instance of org.apache.cassandra.security.ISslContextFactory
+  #     class_name: org.apache.cassandra.security.DefaultSslContextFactory
+  # Verify peer server certificates
+  require_client_auth: false
+  # Set to a valid trustore if require_client_auth is true
+  truststore: conf/.truststore
+  truststore_password: cassandra
+  # Verify that the host name in the certificate matches the connected host
+  require_endpoint_verification: false
+  # More advanced defaults:
+  # protocol: TLS
+  # store_type: JKS
+  # cipher_suites: [
+  #   TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+  #   TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+  #   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA,
+  #   TLS_RSA_WITH_AES_256_CBC_SHA
+  # ]
+
+# Configure client-to-server encryption.
+#
+# **NOTE** this default configuration is an insecure configuration. If you need to
+# enable client-to-server encryption generate server keystores (and truststores for mutual
+# authentication) per:
+# http://download.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
+# Then perform the following configuration changes:
+#
+# Step 1: Set enabled=true and explicitly set optional=true. Restart all nodes
+#
+# Step 2: Set optional=false (or remove it) and if you generated truststores and want to use mutual
+# auth set require_client_auth=true. Restart all nodes
+client_encryption_options:
+  # Enable client-to-server encryption
+  enabled: false
+  # When set to true, encrypted and unencrypted connections are allowed on the native_transport_port
+  # This should _only be true_ while in unencrypted or transitional operation
+  # optional defaults to true when enabled is false, and false when enabled is true.
+  # optional: true
+  # Set keystore and keystore_password to valid keystores if enabled is true
+  keystore: conf/.keystore
+  keystore_password: cassandra
+  # Configure the way Cassandra creates SSL contexts.
+  # To use PEM-based key material, see org.apache.cassandra.security.PEMBasedSslContextFactory
+  # ssl_context_factory:
+  #     # Must be an instance of org.apache.cassandra.security.ISslContextFactory
+  #     class_name: org.apache.cassandra.security.DefaultSslContextFactory
+  # Verify client certificates
+  require_client_auth: false
+  # Set trustore and truststore_password if require_client_auth is true
+  # truststore: conf/.truststore
+  # truststore_password: cassandra
+  # More advanced defaults:
+  # protocol: TLS
+  # store_type: JKS
+  # cipher_suites: [
+  #   TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+  #   TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+  #   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA,
+  #   TLS_RSA_WITH_AES_256_CBC_SHA
+  # ]
+
+# internode_compression controls whether traffic between nodes is
+# compressed.
+# Can be:
+#
+# all
+#   all traffic is compressed
+#
+# dc
+#   traffic between different datacenters is compressed
+#
+# none
+#   nothing is compressed.
+internode_compression: dc
+
+# Enable or disable tcp_nodelay for inter-dc communication.
+# Disabling it will result in larger (but fewer) network packets being sent,
+# reducing overhead from the TCP protocol itself, at the cost of increasing
+# latency if you block for cross-datacenter responses.
+inter_dc_tcp_nodelay: false
+
+# TTL for different trace types used during logging of the repair process.
+# Min unit: s
+trace_type_query_ttl: 1d
+# Min unit: s
+trace_type_repair_ttl: 7d
+
+# If unset, all GC Pauses greater than gc_log_threshold will log at
+# INFO level
+# UDFs (user defined functions) are disabled by default.
+# As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code.
+user_defined_functions_enabled: false
+
+# Enables scripted UDFs (JavaScript UDFs).
+# Java UDFs are always enabled, if user_defined_functions_enabled is true.
+# Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider.
+# This option has no effect, if user_defined_functions_enabled is false.
+scripted_user_defined_functions_enabled: false
+
+# Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from
+# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by
+# the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys
+# can still (and should!) be in the keystore and will be used on decrypt operations
+# (to handle the case of key rotation).
+#
+# It is strongly recommended to download and install Java Cryptography Extension (JCE)
+# Unlimited Strength Jurisdiction Policy Files for your version of the JDK.
+# (current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html)
+#
+# Currently, only the following file types are supported for transparent data encryption, although
+# more are coming in future cassandra releases: commitlog, hints
+transparent_data_encryption_options:
+  enabled: false
+  chunk_length_kb: 64
+  cipher: AES/CBC/PKCS5Padding
+  key_alias: testing:1
+  # CBC IV length for AES needs to be 16 bytes (which is also the default size)
+  # iv_length: 16
+  key_provider:
+    - class_name: org.apache.cassandra.security.JKSKeyProvider
+      parameters:
+        - keystore: conf/.keystore
+          keystore_password: cassandra
+          store_type: JCEKS
+          key_password: cassandra
+
+
+#####################
+# SAFETY THRESHOLDS #
+#####################
+
+# When executing a scan, within or across a partition, we need to keep the
+# tombstones seen in memory so we can return them to the coordinator, which
+# will use them to make sure other replicas also know about the deleted rows.
+# With workloads that generate a lot of tombstones, this can cause performance
+# problems and even exaust the server heap.
+# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
+# Adjust the thresholds here if you understand the dangers and want to
+# scan more tombstones anyway.  These thresholds may also be adjusted at runtime
+# using the StorageService mbean.
+tombstone_warn_threshold: 1000
+tombstone_failure_threshold: 100000
+
+# Filtering and secondary index queries at read consistency levels above ONE/LOCAL_ONE use a
+# mechanism called replica filtering protection to ensure that results from stale replicas do
+# not violate consistency. (See CASSANDRA-8272 and CASSANDRA-15907 for more details.) This
+# mechanism materializes replica results by partition on-heap at the coordinator. The more possibly
+# stale results returned by the replicas, the more rows materialized during the query.
+replica_filtering_protection:
+    # These thresholds exist to limit the damage severely out-of-date replicas can cause during these
+    # queries. They limit the number of rows from all replicas individual index and filtering queries
+    # can materialize on-heap to return correct results at the desired read consistency level.
+    #
+    # "cached_replica_rows_warn_threshold" is the per-query threshold at which a warning will be logged.
+    # "cached_replica_rows_fail_threshold" is the per-query threshold at which the query will fail.
+    #
+    # These thresholds may also be adjusted at runtime using the StorageService mbean.
+    #
+    # If the failure threshold is breached, it is likely that either the current page/fetch size
+    # is too large or one or more replicas is severely out-of-sync and in need of repair.
+    cached_rows_warn_threshold: 2000
+    cached_rows_fail_threshold: 32000
+
+# Log WARN on any multiple-partition batch size exceeding this value. 5KiB per batch by default.
+# Caution should be taken on increasing the size of this threshold as it can lead to node instability.
+# Min unit: KiB
+batch_size_warn_threshold: 5KiB
+
+# Fail any multiple-partition batch exceeding this value. 50KiB (10x warn threshold) by default.
+# Min unit: KiB
+batch_size_fail_threshold: 50KiB
+
+# Log WARN on any batches not of type LOGGED than span across more partitions than this limit
+unlogged_batch_across_partitions_warn_threshold: 10
+
+# Log a warning when compacting partitions larger than this value
+compaction_large_partition_warning_threshold: 100MiB
+
+# Log a warning when writing more tombstones than this value to a partition
+compaction_tombstone_warning_threshold: 100000
+
+# GC Pauses greater than 200 ms will be logged at INFO level
+# This threshold can be adjusted to minimize logging if necessary
+# Min unit: ms
+# gc_log_threshold: 200ms
+
+# GC Pauses greater than gc_warn_threshold will be logged at WARN level
+# Adjust the threshold based on your application throughput requirement. Setting to 0
+# will deactivate the feature.
+# Min unit: ms
+# gc_warn_threshold: 1000ms
+
+# Maximum size of any value in SSTables. Safety measure to detect SSTable corruption
+# early. Any value size larger than this threshold will result into marking an SSTable
+# as corrupted. This should be positive and less than 2GiB.
+# Min unit: MiB
+# max_value_size: 256MiB
+
+# ** Impact on keyspace creation **
+# If replication factor is not mentioned as part of keyspace creation, default_keyspace_rf would apply.
+# Changing this configuration would only take effect for keyspaces created after the change, but does not impact
+# existing keyspaces created prior to the change.
+# ** Impact on keyspace alter **
+# When altering a keyspace from NetworkTopologyStrategy to SimpleStrategy, default_keyspace_rf is applied if rf is not
+# explicitly mentioned.
+# ** Impact on system keyspaces **
+# This would also apply for any system keyspaces that need replication factor.
+# A further note about system keyspaces - system_traces and system_distributed keyspaces take RF of 2 or default,
+# whichever is higher, and system_auth keyspace takes RF of 1 or default, whichever is higher.
+# Suggested value for use in production: 3
+# default_keyspace_rf: 1
+
+# Track a metric per keyspace indicating whether replication achieved the ideal consistency
+# level for writes without timing out. This is different from the consistency level requested by
+# each write which may be lower in order to facilitate availability.
+# ideal_consistency_level: EACH_QUORUM
+
+# Automatically upgrade sstables after upgrade - if there is no ordinary compaction to do, the
+# oldest non-upgraded sstable will get upgraded to the latest version
+# automatic_sstable_upgrade: false
+# Limit the number of concurrent sstable upgrades
+# max_concurrent_automatic_sstable_upgrades: 1
+
+# Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs
+# on audit_logging for full details about the various configuration options.
+audit_logging_options:
+  enabled: false
+  logger:
+    - class_name: BinAuditLogger
+  # audit_logs_dir:
+  # included_keyspaces:
+  # excluded_keyspaces: system, system_schema, system_virtual_schema
+  # included_categories:
+  # excluded_categories:
+  # included_users:
+  # excluded_users:
+  # roll_cycle: HOURLY
+  # block: true
+  # max_queue_weight: 268435456 # 256 MiB
+  # max_log_size: 17179869184 # 16 GiB
+  ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
+  # archive_command:
+  # max_archive_retries: 10
+
+
+# default options for full query logging - these can be overridden from command line when executing
+# nodetool enablefullquerylog
+# full_query_logging_options:
+  # log_dir:
+  # roll_cycle: HOURLY
+  # block: true
+  # max_queue_weight: 268435456 # 256 MiB
+  # max_log_size: 17179869184 # 16 GiB
+  ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled:
+  # archive_command:
+  ## note that enabling this allows anyone with JMX/nodetool access to run local shell commands as the user running cassandra
+  # allow_nodetool_archive_command: false
+  # max_archive_retries: 10
+
+# validate tombstones on reads and compaction
+# can be either "disabled", "warn" or "exception"
+# corrupted_tombstone_strategy: disabled
+
+# Diagnostic Events #
+# If enabled, diagnostic events can be helpful for troubleshooting operational issues. Emitted events contain details
+# on internal state and temporal relationships across events, accessible by clients via JMX.
+diagnostic_events_enabled: false
+
+# Use native transport TCP message coalescing. If on upgrade to 4.0 you found your throughput decreasing, and in
+# particular you run an old kernel or have very fewer client connections, this option might be worth evaluating.
+#native_transport_flush_in_batches_legacy: false
+
+# Enable tracking of repaired state of data during reads and comparison between replicas
+# Mismatches between the repaired sets of replicas can be characterized as either confirmed
+# or unconfirmed. In this context, unconfirmed indicates that the presence of pending repair
+# sessions, unrepaired partition tombstones, or some other condition means that the disparity
+# cannot be considered conclusive. Confirmed mismatches should be a trigger for investigation
+# as they may be indicative of corruption or data loss.
+# There are separate flags for range vs partition reads as single partition reads are only tracked
+# when CL > 1 and a digest mismatch occurs. Currently, range queries don't use digests so if
+# enabled for range reads, all range reads will include repaired data tracking. As this adds
+# some overhead, operators may wish to disable it whilst still enabling it for partition reads
+repaired_data_tracking_for_range_reads_enabled: false
+repaired_data_tracking_for_partition_reads_enabled: false
+# If false, only confirmed mismatches will be reported. If true, a separate metric for unconfirmed
+# mismatches will also be recorded. This is to avoid potential signal:noise issues are unconfirmed
+# mismatches are less actionable than confirmed ones.
+report_unconfirmed_repaired_data_mismatches: false
+
+# Having many tables and/or keyspaces negatively affects performance of many operations in the
+# cluster. When the number of tables/keyspaces in the cluster exceeds the following thresholds
+# a client warning will be sent back to the user when creating a table or keyspace.
+# As of cassandra 4.1, these properties are deprecated in favor of keyspaces_warn_threshold and tables_warn_threshold
+# table_count_warn_threshold: 150
+# keyspace_count_warn_threshold: 40
+
+# configure the read and write consistency levels for modifications to auth tables
+# auth_read_consistency_level: LOCAL_QUORUM
+# auth_write_consistency_level: EACH_QUORUM
+
+# Delays on auth resolution can lead to a thundering herd problem on reconnects; this option will enable
+# warming of auth caches prior to node completing startup. See CASSANDRA-16958
+# auth_cache_warming_enabled: false
+
+#########################
+# EXPERIMENTAL FEATURES #
+#########################
+
+# Enables materialized view creation on this node.
+# Materialized views are considered experimental and are not recommended for production use.
+materialized_views_enabled: true
+
+# Enables SASI index creation on this node.
+# SASI indexes are considered experimental and are not recommended for production use.
+sasi_indexes_enabled: false
+
+# Enables creation of transiently replicated keyspaces on this node.
+# Transient replication is experimental and is not recommended for production use.
+transient_replication_enabled: false
+
+# Enables the used of 'ALTER ... DROP COMPACT STORAGE' statements on this node.
+# 'ALTER ... DROP COMPACT STORAGE' is considered experimental and is not recommended for production use.
+drop_compact_storage_enabled: false
+
+# Whether or not USE <keyspace> is allowed. This is enabled by default to avoid failure on upgrade.
+#use_statements_enabled: true
+
+# When the client triggers a protocol exception or unknown issue (Cassandra bug) we increment
+# a client metric showing this; this logic will exclude specific subnets from updating these
+# metrics
+#client_error_reporting_exclusions:
+#  subnets:
+#    - 127.0.0.1
+#    - 127.0.0.0/31
+
+# Enables read thresholds (warn/fail) across all replicas for reporting back to the client.
+# See: CASSANDRA-16850
+# read_thresholds_enabled: false # scheduled to be set true in 4.2
+# When read_thresholds_enabled: true, this tracks the materialized size of a query on the
+# coordinator. If coordinator_read_size_warn_threshold is defined, this will emit a warning
+# to clients with details on what query triggered this as well as the size of the result set; if
+# coordinator_read_size_fail_threshold is defined, this will fail the query after it
+# has exceeded this threshold, returning a read error to the user.
+# coordinator_read_size_warn_threshold:
+# coordinator_read_size_fail_threshold:
+# When read_thresholds_enabled: true, this tracks the size of the local read (as defined by
+# heap size), and will warn/fail based off these thresholds; undefined disables these checks.
+# local_read_size_warn_threshold:
+# local_read_size_fail_threshold:
+# When read_thresholds_enabled: true, this tracks the expected memory size of the RowIndexEntry
+# and will warn/fail based off these thresholds; undefined disables these checks
+# row_index_read_size_warn_threshold:
+# row_index_read_size_fail_threshold:
+
+# Guardrail to warn or fail when creating more user keyspaces than threshold.
+# The two thresholds default to -1 to disable.
+# keyspaces_warn_threshold: -1
+# keyspaces_fail_threshold: -1
+# Guardrail to warn or fail when creating more user tables than threshold.
+# The two thresholds default to -1 to disable.
+# tables_warn_threshold: -1
+# tables_fail_threshold: -1
+# Guardrail to enable or disable the ability to create uncompressed tables
+# uncompressed_tables_enabled: true
+# Guardrail to warn or fail when creating/altering a table with more columns per table than threshold.
+# The two thresholds default to -1 to disable.
+# columns_per_table_warn_threshold: -1
+# columns_per_table_fail_threshold: -1
+# Guardrail to warn or fail when creating more secondary indexes per table than threshold.
+# The two thresholds default to -1 to disable.
+# secondary_indexes_per_table_warn_threshold: -1
+# secondary_indexes_per_table_fail_threshold: -1
+# Guardrail to enable or disable the creation of secondary indexes
+# secondary_indexes_enabled: true
+# Guardrail to warn or fail when creating more materialized views per table than threshold.
+# The two thresholds default to -1 to disable.
+# materialized_views_per_table_warn_threshold: -1
+# materialized_views_per_table_fail_threshold: -1
+# Guardrail to warn about, ignore or reject properties when creating tables. By default all properties are allowed.
+# table_properties_warned: []
+# table_properties_ignored: []
+# table_properties_disallowed: []
+# Guardrail to allow/disallow user-provided timestamps. Defaults to true.
+# user_timestamps_enabled: true
+# Guardrail to allow/disallow GROUP BY functionality.
+# group_by_enabled: true
+# Guardrail to allow/disallow TRUNCATE and DROP TABLE statements
+# drop_truncate_table_enabled: true
+# Guardrail to warn or fail when using a page size greater than threshold.
+# The two thresholds default to -1 to disable.
+# page_size_warn_threshold: -1
+# page_size_fail_threshold: -1
+# Guardrail to allow/disallow list operations that require read before write, i.e. setting list element by index and
+# removing list elements by either index or value. Defaults to true.
+# read_before_write_list_operations_enabled: true
+# Guardrail to warn or fail when querying with an IN restriction selecting more partition keys than threshold.
+# The two thresholds default to -1 to disable.
+# partition_keys_in_select_warn_threshold: -1
+# partition_keys_in_select_fail_threshold: -1
+# Guardrail to warn or fail when an IN query creates a cartesian product with a size exceeding threshold,
+# eg. "a in (1,2,...10) and b in (1,2...10)" results in cartesian product of 100.
+# The two thresholds default to -1 to disable.
+# in_select_cartesian_product_warn_threshold: -1
+# in_select_cartesian_product_fail_threshold: -1
+# Guardrail to warn about or reject read consistency levels. By default, all consistency levels are allowed.
+# read_consistency_levels_warned: []
+# read_consistency_levels_disallowed: []
+# Guardrail to warn about or reject write consistency levels. By default, all consistency levels are allowed.
+# write_consistency_levels_warned: []
+# write_consistency_levels_disallowed: []
+# Guardrail to warn or fail when encountering larger size of collection data than threshold.
+# At query time this guardrail is applied only to the collection fragment that is being writen, even though in the case
+# of non-frozen collections there could be unaccounted parts of the collection on the sstables. This is done this way to
+# prevent read-before-write. The guardrail is also checked at sstable write time to detect large non-frozen collections,
+# although in that case exceeding the fail threshold will only log an error message, without interrupting the operation.
+# The two thresholds default to null to disable.
+# Min unit: B
+# collection_size_warn_threshold:
+# Min unit: B
+# collection_size_fail_threshold:
+# Guardrail to warn or fail when encountering more elements in collection than threshold.
+# At query time this guardrail is applied only to the collection fragment that is being writen, even though in the case
+# of non-frozen collections there could be unaccounted parts of the collection on the sstables. This is done this way to
+# prevent read-before-write. The guardrail is also checked at sstable write time to detect large non-frozen collections,
+# although in that case exceeding the fail threshold will only log an error message, without interrupting the operation.
+# The two thresholds default to -1 to disable.
+# items_per_collection_warn_threshold: -1
+# items_per_collection_fail_threshold: -1
+# Guardrail to allow/disallow querying with ALLOW FILTERING. Defaults to true.
+# allow_filtering_enabled: true
+# Guardrail to warn or fail when creating a user-defined-type with more fields in than threshold.
+# Default -1 to disable.
+# fields_per_udt_warn_threshold: -1
+# fields_per_udt_fail_threshold: -1
+# Guardrail to warn or fail when local data disk usage percentage exceeds threshold. Valid values are in [1, 100].
+# This is only used for the disks storing data directories, so it won't count any separate disks used for storing
+# the commitlog, hints nor saved caches. The disk usage is the ratio between the amount of space used by the data
+# directories and the addition of that same space and the remaining free space on disk. The main purpose of this
+# guardrail is rejecting user writes when the disks are over the defined usage percentage, so the writes done by
+# background processes such as compaction and streaming don't fail due to a full disk. The limits should be defined
+# accordingly to the expected data growth due to those background processes, so for example a compaction strategy
+# doubling the size of the data would require to keep the disk usage under 50%.
+# The two thresholds default to -1 to disable.
+# data_disk_usage_percentage_warn_threshold: -1
+# data_disk_usage_percentage_fail_threshold: -1
+# Allows defining the max disk size of the data directories when calculating thresholds for
+# disk_usage_percentage_warn_threshold and disk_usage_percentage_fail_threshold, so if this is greater than zero they
+# become percentages of a fixed size on disk instead of percentages of the physically available disk size. This should
+# be useful when we have a large disk and we only want to use a part of it for Cassandra's data directories.
+# Valid values are in [1, max available disk size of all data directories].
+# Defaults to null to disable and use the physically available disk size of data directories during calculations.
+# Min unit: B
+# data_disk_usage_max_disk_size:
+# Guardrail to warn or fail when the minimum replication factor is lesser than threshold.
+# This would also apply to system keyspaces.
+# Suggested value for use in production: 2 or higher
+# minimum_replication_factor_warn_threshold: -1
+# minimum_replication_factor_fail_threshold: -1
+
+# Startup Checks are executed as part of Cassandra startup process, not all of them
+# are configurable (so you can disable them) but these which are enumerated bellow.
+# Uncomment the startup checks and configure them appropriately to cover your needs.
+#
+#startup_checks:
+# Verifies correct ownership of attached locations on disk at startup. See CASSANDRA-16879 for more details.
+#  check_filesystem_ownership:
+#    enabled: false
+#    ownership_token: "sometoken" # (overriden by "CassandraOwnershipToken" system property)
+#    ownership_filename: ".cassandra_fs_ownership" # (overriden by "cassandra.fs_ownership_filename")
+# Prevents a node from starting if snitch's data center differs from previous data center.
+#  check_dc:
+#    enabled: true # (overriden by cassandra.ignore_dc system property)
+# Prevents a node from starting if snitch's rack differs from previous rack.
+#  check_rack:
+#    enabled: true # (overriden by cassandra.ignore_rack system property)
+# Enable this property to fail startup if the node is down for longer than gc_grace_seconds, to potentially
+# prevent data resurrection on tables with deletes. By default, this will run against all keyspaces and tables
+# except the ones specified on excluded_keyspaces and excluded_tables.
+#  check_data_resurrection:
+#    enabled: false
+# file where Cassandra periodically writes the last time it was known to run
+#    heartbeat_file: /var/lib/cassandra/data/cassandra-heartbeat
+#    excluded_keyspaces: # comma separated list of keyspaces to exclude from the check
+#    excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check
diff --git a/metadata-ingestion/tests/integration/cassandra/setup/init_keyspaces.cql b/metadata-ingestion/tests/integration/cassandra/setup/init_keyspaces.cql
new file mode 100644
index 0000000000000..81e919461a55b
--- /dev/null
+++ b/metadata-ingestion/tests/integration/cassandra/setup/init_keyspaces.cql
@@ -0,0 +1,131 @@
+
+CREATE KEYSPACE cass_test_1 WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};
+CREATE KEYSPACE cass_test_2 WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};
+
+CREATE TABLE cass_test_1.information (
+    person_id int PRIMARY KEY,
+    last_updated timestamp,
+    details text
+);
+
+CREATE TABLE cass_test_1.people (
+    person_id int PRIMARY KEY,
+    name text,
+    email text
+);
+
+CREATE TABLE cass_test_2.tasks (
+    task_id int PRIMARY KEY,
+    last_updated timestamp,
+    details text,
+    status text
+);
+
+CREATE MATERIALIZED VIEW cass_test_2.task_status AS
+SELECT
+    task_id,
+    status
+FROM cass_test_2.tasks
+WHERE status IS NOT NULL AND task_id IS NOT NULL
+PRIMARY KEY (task_id, status);
+
+-- Create Keyspace with comments
+CREATE KEYSPACE IF NOT EXISTS example_keyspace
+WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
+
+-- Use Keyspace
+USE example_keyspace;
+
+-- Table with non-counter column types
+CREATE TABLE IF NOT EXISTS all_data_types (
+    id uuid PRIMARY KEY,
+    ascii_column ascii,
+    bigint_column bigint,
+    blob_column blob,
+    boolean_column boolean,
+    date_column date,
+    decimal_column decimal,
+    double_column double,
+    float_column float,
+    inet_column inet,
+    int_column int,
+    list_column list<text>,
+    map_column map<text, text>,
+    set_column set<text>,
+    smallint_column smallint,
+    text_column text,
+    time_column time,
+    timestamp_column timestamp,
+    timeuuid_column timeuuid,
+    tinyint_column tinyint,
+    tuple_column tuple<int, text>,
+    uuid_column uuid,
+    varchar_column varchar,
+    varint_column varint,
+    frozen_map_column frozen<map<text, text>>,
+    frozen_list_column frozen<list<text>>,
+    frozen_set_column frozen<set<text>>
+) WITH COMMENT = 'Table containing all supported Cassandra data types, excluding counters';
+
+-- Separate table for counters
+CREATE TABLE IF NOT EXISTS counter_table (
+    id uuid PRIMARY KEY,
+    counter_column counter
+) WITH COMMENT = 'Separate table containing only counter column';
+
+-- Sample view
+CREATE MATERIALIZED VIEW IF NOT EXISTS example_view_1 AS
+    SELECT id, ascii_column, bigint_column
+    FROM all_data_types
+    WHERE id IS NOT NULL AND ascii_column IS NOT NULL
+    PRIMARY KEY (id, ascii_column) WITH COMMENT = 'Example view definition with id and ascii_column';
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS example_view_2 AS
+    SELECT id, ascii_column, float_column
+    FROM all_data_types
+    WHERE id IS NOT NULL AND ascii_column IS NOT NULL
+    PRIMARY KEY (id, ascii_column) WITH COMMENT = 'Example view definition with id and ascii_column';
+
+-- Table created for profilling
+CREATE TABLE IF NOT EXISTS shopping_cart (
+userid text PRIMARY KEY,
+item_count int,
+last_update_timestamp timestamp
+);
+
+-- Insert some data
+INSERT INTO shopping_cart
+(userid, item_count, last_update_timestamp)
+VALUES ('9876', 2, '2024-11-01T00:00:00.000+0000');
+
+INSERT INTO shopping_cart
+(userid, item_count, last_update_timestamp)
+VALUES ('1234', 5, '2024-11-02T00:00:00.000+0000');
+
+INSERT INTO shopping_cart
+(userid, item_count, last_update_timestamp)
+VALUES ('1235', 100, '2024-11-03T00:00:00.000+0000');
+
+INSERT INTO shopping_cart
+(userid, item_count, last_update_timestamp)
+VALUES ('1236', 50, '2024-11-04T00:00:00.000+0000');
+
+INSERT INTO shopping_cart
+(userid, item_count, last_update_timestamp)
+VALUES ('1237', 75, '2024-11-05T00:00:00.000+0000');
+
+INSERT INTO shopping_cart
+(userid, last_update_timestamp)
+VALUES ('1238', '2024-11-06T00:00:00.000+0000');
+
+INSERT INTO shopping_cart
+(userid, last_update_timestamp)
+VALUES ('1239', '2024-11-07T00:00:00.000+0000');
+
+INSERT INTO shopping_cart
+(userid, last_update_timestamp)
+VALUES ('1240', '2024-11-08T00:00:00.000+0000');
+
+INSERT INTO shopping_cart
+(userid, last_update_timestamp)
+VALUES ('1241', '2024-11-09T00:00:00.000+0000');
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/cassandra/test_cassandra.py b/metadata-ingestion/tests/integration/cassandra/test_cassandra.py
new file mode 100644
index 0000000000000..d561308aaad20
--- /dev/null
+++ b/metadata-ingestion/tests/integration/cassandra/test_cassandra.py
@@ -0,0 +1,53 @@
+import logging
+import time
+
+import pytest
+
+from datahub.ingestion.run.pipeline import Pipeline
+from tests.test_helpers import mce_helpers
+from tests.test_helpers.docker_helpers import wait_for_port
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.integration
+def test_cassandra_ingest(docker_compose_runner, pytestconfig, tmp_path):
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/cassandra"
+
+    with docker_compose_runner(
+        test_resources_dir / "docker-compose.yml", "cassandra"
+    ) as docker_services:
+        wait_for_port(docker_services, "test-cassandra", 9042)
+
+        time.sleep(5)
+        # Run the metadata ingestion pipeline.
+        logger.info("Starting the ingestion test...")
+        pipeline_default_platform_instance = Pipeline.create(
+            {
+                "run_id": "cassandra-test",
+                "source": {
+                    "type": "cassandra",
+                    "config": {
+                        "contact_point": "localhost",
+                        "port": 9042,
+                        "profiling": {"enabled": True},
+                    },
+                },
+                "sink": {
+                    "type": "file",
+                    "config": {
+                        "filename": f"{tmp_path}/cassandra_mcps.json",
+                    },
+                },
+            }
+        )
+        pipeline_default_platform_instance.run()
+        pipeline_default_platform_instance.raise_from_status()
+
+        # Verify the output.
+        logger.info("Verifying output.")
+        mce_helpers.check_golden_file(
+            pytestconfig,
+            output_path=f"{tmp_path}/cassandra_mcps.json",
+            golden_path=test_resources_dir / "cassandra_mcps_golden.json",
+        )
diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py
index 390d8d7698dd4..c6a3dc4fd590b 100644
--- a/metadata-ingestion/tests/integration/dbt/test_dbt.py
+++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py
@@ -11,12 +11,6 @@
 from datahub.ingestion.run.pipeline_config import PipelineConfig, SourceConfig
 from datahub.ingestion.source.dbt.dbt_common import DBTEntitiesEnabled, EmitDirective
 from datahub.ingestion.source.dbt.dbt_core import DBTCoreConfig, DBTCoreSource
-from datahub.ingestion.source.sql.sql_types import (
-    ATHENA_SQL_TYPES_MAP,
-    TRINO_SQL_TYPES_MAP,
-    resolve_athena_modified_type,
-    resolve_trino_modified_type,
-)
 from tests.test_helpers import mce_helpers, test_connection_helpers
 
 FROZEN_TIME = "2022-02-03 07:00:00"
@@ -362,69 +356,6 @@ def test_dbt_tests(test_resources_dir, pytestconfig, tmp_path, mock_time, **kwar
     )
 
 
-@pytest.mark.parametrize(
-    "data_type, expected_data_type",
-    [
-        ("boolean", "boolean"),
-        ("tinyint", "tinyint"),
-        ("smallint", "smallint"),
-        ("int", "int"),
-        ("integer", "integer"),
-        ("bigint", "bigint"),
-        ("real", "real"),
-        ("double", "double"),
-        ("decimal(10,0)", "decimal"),
-        ("varchar(20)", "varchar"),
-        ("char", "char"),
-        ("varbinary", "varbinary"),
-        ("json", "json"),
-        ("date", "date"),
-        ("time", "time"),
-        ("time(12)", "time"),
-        ("timestamp", "timestamp"),
-        ("timestamp(3)", "timestamp"),
-        ("row(x bigint, y double)", "row"),
-        ("array(row(x bigint, y double))", "array"),
-        ("map(varchar, varchar)", "map"),
-    ],
-)
-def test_resolve_trino_modified_type(data_type, expected_data_type):
-    assert (
-        resolve_trino_modified_type(data_type)
-        == TRINO_SQL_TYPES_MAP[expected_data_type]
-    )
-
-
-@pytest.mark.parametrize(
-    "data_type, expected_data_type",
-    [
-        ("boolean", "boolean"),
-        ("tinyint", "tinyint"),
-        ("smallint", "smallint"),
-        ("int", "int"),
-        ("integer", "integer"),
-        ("bigint", "bigint"),
-        ("float", "float"),
-        ("double", "double"),
-        ("decimal(10,0)", "decimal"),
-        ("varchar(20)", "varchar"),
-        ("char", "char"),
-        ("binary", "binary"),
-        ("date", "date"),
-        ("timestamp", "timestamp"),
-        ("timestamp(3)", "timestamp"),
-        ("struct<x timestamp(3), y timestamp>", "struct"),
-        ("array<struct<x bigint, y double>>", "array"),
-        ("map<varchar, varchar>", "map"),
-    ],
-)
-def test_resolve_athena_modified_type(data_type, expected_data_type):
-    assert (
-        resolve_athena_modified_type(data_type)
-        == ATHENA_SQL_TYPES_MAP[expected_data_type]
-    )
-
-
 @pytest.mark.integration
 @freeze_time(FROZEN_TIME)
 def test_dbt_tests_only_assertions(
diff --git a/metadata-ingestion/tests/integration/dremio/docker-compose.yml b/metadata-ingestion/tests/integration/dremio/docker-compose.yml
new file mode 100644
index 0000000000000..500e8bbcef729
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/docker-compose.yml
@@ -0,0 +1,56 @@
+version: "3"
+
+services:
+  # Minio Storage Server
+  minio:
+    image: minio/minio:RELEASE.2023-07-21T21-12-44Z
+    container_name: minio
+    environment:
+      - MINIO_ROOT_USER=miniouser
+      - MINIO_ROOT_PASSWORD=miniopassword
+      - MINIO_DOMAIN=storage
+      - MINIO_REGION_NAME=us-east-1
+      - MINIO_REGION=us-east-1
+    networks:
+      dremio-network:
+    ports:
+      - 9001:9001
+      - 9000:9000
+    command: ["server", "/data", "--console-address", ":9001"]
+
+  #MySql
+  mysqldb:
+    image: mysql:8.0
+    environment:
+      MYSQL_ROOT_PASSWORD: rootpwd123
+      MYSQL_USER: foo
+      MYSQL_PASSWORD: datahub123
+    container_name: test_mysql
+    hostname: test_mysql
+    ports:
+      - "23306:3306"
+    volumes:
+      - ./setup/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf
+      - ./setup/setup.sql:/docker-entrypoint-initdb.d/setup.sql
+    networks:
+      dremio-network:
+
+  # Dremio
+  dremio:
+    platform: linux/x86_64
+    image: dremio/dremio-oss:latest
+    container_name: dremio
+    ports:
+      - 9047:9047
+      - 31010:31010
+      - 32010:32010
+    environment:
+      - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dpaths.dist=file:///opt/dremio/data/dist
+    networks:
+      dremio-network:
+    depends_on:
+      - minio
+      - mysqldb
+
+networks:
+  dremio-network:
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json
new file mode 100644
index 0000000000000..41d47a574a211
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json
@@ -0,0 +1,8806 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "s3",
+            "qualifiedName": "s3",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "mysql",
+            "qualifiedName": "mysql",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "Samples",
+            "qualifiedName": "Samples",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "@admin",
+            "qualifiedName": "@admin",
+            "description": "#  Wikis & Labels\n\n![Gnarly Catalog](https://d33wubrfki0l68.cloudfront.net/c1a54376c45a9276c080f3d10ed25ce61c17bcd2/2b946/img/home/open-source-for-everyone.svg)\n\nYou are reading the wiki for your home space! You can create and edit this information for any source, space, or folder.\n\nThis sidebar always shows the wiki for the current source, space or folder you are browsing.\n\nWhen browsing or previewing datasets, click on the `Open details panel` button to create a wiki or add labels to that dataset.\n\n**Tip:** You can hide the wiki by clicking on the sidebar icon on upper right hand side.",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Space"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "space",
+            "qualifiedName": "space",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Space"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "test_folder",
+            "qualifiedName": "space.test_folder",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+                    "urn": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ba5e0b6f1fb9b5c6c535d55a464ab5c5",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "dataCharmer",
+            "qualifiedName": "mysql.dataCharmer",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ba5e0b6f1fb9b5c6c535d55a464ab5c5",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ba5e0b6f1fb9b5c6c535d55a464ab5c5",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ba5e0b6f1fb9b5c6c535d55a464ab5c5",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ba5e0b6f1fb9b5c6c535d55a464ab5c5",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ba5e0b6f1fb9b5c6c535d55a464ab5c5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f280657a9759203790c7755104f03e52",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "metagalaxy",
+            "qualifiedName": "mysql.metagalaxy",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f280657a9759203790c7755104f03e52",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f280657a9759203790c7755104f03e52",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f280657a9759203790c7755104f03e52",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f280657a9759203790c7755104f03e52",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f280657a9759203790c7755104f03e52",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:811317054d9b6934fb2df52c8ecad37f",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "mysql",
+            "qualifiedName": "mysql.mysql",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:811317054d9b6934fb2df52c8ecad37f",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:811317054d9b6934fb2df52c8ecad37f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:811317054d9b6934fb2df52c8ecad37f",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:811317054d9b6934fb2df52c8ecad37f",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:811317054d9b6934fb2df52c8ecad37f",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "northwind",
+            "qualifiedName": "mysql.northwind",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c99ddc4a9bb768e6b2136f78609c520a",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "performance_schema",
+            "qualifiedName": "mysql.performance_schema",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c99ddc4a9bb768e6b2136f78609c520a",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c99ddc4a9bb768e6b2136f78609c520a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c99ddc4a9bb768e6b2136f78609c520a",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c99ddc4a9bb768e6b2136f78609c520a",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c99ddc4a9bb768e6b2136f78609c520a",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:90b7c77f3141af8db284348c2a5ce2bd",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "test_cases",
+            "qualifiedName": "mysql.test_cases",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:90b7c77f3141af8db284348c2a5ce2bd",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:90b7c77f3141af8db284348c2a5ce2bd",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:90b7c77f3141af8db284348c2a5ce2bd",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:90b7c77f3141af8db284348c2a5ce2bd",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:90b7c77f3141af8db284348c2a5ce2bd",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "samples.dremio.com",
+            "qualifiedName": "Samples.samples.dremio.com",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "Dremio University",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "tpcds_sf1000",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "catalog_page",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+                    "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+                    "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+                },
+                {
+                    "id": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+                    "urn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"warehouse\"",
+            "name": "warehouse",
+            "qualifiedName": "space.warehouse",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * from Samples.\"samples.dremio.com\".\"NYC-weather.csv\"",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.warehouse",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "A",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "B",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "C",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "D",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "E",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "F",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "G",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "H",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "I",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+                    "urn": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"customers\"",
+            "name": "customers",
+            "qualifiedName": "space.test_folder.customers",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.northwind.customers",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.customers",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "priority",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float(24)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "email_address",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "first_name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "last_name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "company",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+                    "urn": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+                },
+                {
+                    "id": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+                    "urn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_aspect\"",
+            "name": "metadata_aspect",
+            "qualifiedName": "space.test_folder.metadata_aspect",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_aspect",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.metadata_aspect",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "aspect",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "version",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "metadata",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdon",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.DateType": {}
+                        }
+                    },
+                    "nativeDataType": "timestamp(23)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdby",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdfor",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+                    "urn": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+                },
+                {
+                    "id": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+                    "urn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_index\"",
+            "name": "metadata_index",
+            "qualifiedName": "space.test_folder.metadata_index",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.metadata_index",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "doubleVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "stringVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "longVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "path",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "aspect",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+                    "urn": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+                },
+                {
+                    "id": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+                    "urn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_index_view\"",
+            "name": "metadata_index_view",
+            "qualifiedName": "space.test_folder.metadata_index_view",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index_view",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.metadata_index_view",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "doubleVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "path",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+                    "urn": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+                },
+                {
+                    "id": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+                    "urn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"orders\"",
+            "name": "orders",
+            "qualifiedName": "space.test_folder.orders",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.northwind.orders",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.orders",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "customer_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "description",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+                    "urn": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+                },
+                {
+                    "id": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+                    "urn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"raw\"",
+            "name": "raw",
+            "qualifiedName": "space.test_folder.raw",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM s3.warehouse",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.raw",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "age",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "salary",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:090caabbf8b73fafa83b546f840bd468",
+                    "urn": "urn:li:container:090caabbf8b73fafa83b546f840bd468"
+                },
+                {
+                    "id": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3",
+                    "urn": "urn:li:container:8f30c87fdc4a3eb093b4558fc58cf2b3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"s3\"/\"warehouse\"",
+            "name": "warehouse",
+            "qualifiedName": "s3.warehouse",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:63a316133b08a091e919dc8c7a828a4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "s3.warehouse",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "age",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "salary",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:63a316133b08a091e919dc8c7a828a4d",
+                    "urn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_aspect\"",
+            "name": "metadata_aspect",
+            "qualifiedName": "mysql.metagalaxy.metadata_aspect",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:f280657a9759203790c7755104f03e52"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.metagalaxy.metadata_aspect",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "version",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "metadata",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdon",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.DateType": {}
+                        }
+                    },
+                    "nativeDataType": "timestamp(23)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdby",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdfor",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "aspect",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                },
+                {
+                    "id": "urn:li:container:f280657a9759203790c7755104f03e52",
+                    "urn": "urn:li:container:f280657a9759203790c7755104f03e52"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_index\"",
+            "name": "metadata_index",
+            "qualifiedName": "mysql.metagalaxy.metadata_index",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:f280657a9759203790c7755104f03e52"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.metagalaxy.metadata_index",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "doubleVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "stringVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "longVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "path",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "aspect",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                },
+                {
+                    "id": "urn:li:container:f280657a9759203790c7755104f03e52",
+                    "urn": "urn:li:container:f280657a9759203790c7755104f03e52"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_index_view\"",
+            "name": "metadata_index_view",
+            "qualifiedName": "mysql.metagalaxy.metadata_index_view",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:f280657a9759203790c7755104f03e52"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.metagalaxy.metadata_index_view",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "path",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "doubleVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                },
+                {
+                    "id": "urn:li:container:f280657a9759203790c7755104f03e52",
+                    "urn": "urn:li:container:f280657a9759203790c7755104f03e52"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"northwind\".\"customers\"",
+            "name": "customers",
+            "qualifiedName": "mysql.northwind.customers",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.northwind.customers",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "priority",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float(24)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "email_address",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "first_name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "last_name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "company",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                },
+                {
+                    "id": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3",
+                    "urn": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"northwind\".\"orders\"",
+            "name": "orders",
+            "qualifiedName": "mysql.northwind.orders",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.northwind.orders",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "description",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "customer_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc411088cb328873d6eba19ee76f09d",
+                    "urn": "urn:li:container:5fc411088cb328873d6eba19ee76f09d"
+                },
+                {
+                    "id": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3",
+                    "urn": "urn:li:container:cc2c44f1f32b6998c445fcd7d6d7a9e3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"NYC-weather.csv\"",
+            "name": "NYC-weather.csv",
+            "qualifiedName": "Samples.samples.dremio.com.NYC-weather.csv",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.NYC-weather.csv",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "D",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "C",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "B",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "A",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "H",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "G",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "F",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "E",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "I",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"googleplaystore.csv\"",
+            "name": "googleplaystore.csv",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "A",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "B",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "C",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "D",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "E",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "F",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "G",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "H",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "I",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "J",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "K",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "L",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "M",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+                    "urn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"oracle-departments.xlsx\"",
+            "name": "oracle-departments.xlsx",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "DEPARTMENT_NAME",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "MANAGER_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "DEPARTMENT_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "LOCATION_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+                    "urn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"tpcds_sf1000\".\"catalog_page\".\"1ab266d5-18eb-4780-711d-0fa337fa6c00\".\"0_0_0.parquet\"",
+            "name": "0_0_0.parquet",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "cp_catalog_page_number",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_type",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_description",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_number",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_end_date_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_start_date_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_page_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_page_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_department",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+                    "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+                },
+                {
+                    "id": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+                    "urn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+                },
+                {
+                    "id": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+                    "urn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),version)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),version)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),metadata)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdon)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdon)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdby)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdfor)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),aspect)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),aspect)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),doubleVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),doubleVal)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),stringVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),longVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),longVal)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),path)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),path)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),aspect)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),aspect)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),path)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),doubleVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),doubleVal)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),priority)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),priority)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),email_address)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),email_address)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),company)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),company)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),description)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),description)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),salary)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.customers,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.northwind.customers",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.customers,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_aspect,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.metagalaxy.metadata_aspect",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_aspect,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.metagalaxy.metadata_index",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index_view,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.metagalaxy.metadata_index_view",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index_view,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.orders,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.northwind.orders",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.orders,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),id)"
+                    ],
+                    "confidenceScore": 0.9,
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),name)"
+                    ],
+                    "confidenceScore": 0.9,
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),age)"
+                    ],
+                    "confidenceScore": 0.9,
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),salary)"
+                    ],
+                    "confidenceScore": 0.9,
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM s3.warehouse",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),id)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),name)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),age)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),salary)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 4,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "doubleVal",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "path",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "urn",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 3,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "description",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "customer_id",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 4,
+            "columnCount": 4,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "age",
+                    "uniqueCount": 4,
+                    "nullCount": 0,
+                    "mean": "32.5",
+                    "stdev": "6.454972243679028"
+                },
+                {
+                    "fieldPath": "name",
+                    "uniqueCount": 4,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 4,
+                    "nullCount": 0,
+                    "mean": "2.5",
+                    "stdev": "1.2909944487358056"
+                },
+                {
+                    "fieldPath": "salary",
+                    "uniqueCount": 4,
+                    "nullCount": 0,
+                    "mean": "65000.0",
+                    "stdev": "12909.944487358056"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 2,
+            "columnCount": 7,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "urn",
+                    "uniqueCount": 1,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "aspect",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "version",
+                    "uniqueCount": 1,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "metadata",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "createdon",
+                    "uniqueCount": 1,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "createdby",
+                    "uniqueCount": 1,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "createdfor",
+                    "uniqueCount": 0,
+                    "nullCount": 2
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 5,
+            "columnCount": 6,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "priority",
+                    "uniqueCount": 3,
+                    "nullCount": 1,
+                    "mean": "4.175000011920929",
+                    "stdev": "0.4924429489953036"
+                },
+                {
+                    "fieldPath": "email_address",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "first_name",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "last_name",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "company",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 5,
+                    "nullCount": 0,
+                    "mean": "3.0",
+                    "stdev": "1.5811388300841898"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 4,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "urn",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "path",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "doubleVal",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 3,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "customer_id",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "description",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 7,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "doubleVal",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "stringVal",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "longVal",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "path",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "aspect",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "urn",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 0,
+            "columnCount": 7,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "doubleVal",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "stringVal",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "longVal",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "path",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "aspect",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "urn",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 0,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 3834,
+            "columnCount": 9,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "D",
+                    "uniqueCount": 76,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "C",
+                    "uniqueCount": 3834,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "B",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "A",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "H",
+                    "uniqueCount": 91,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "G",
+                    "uniqueCount": 40,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "F",
+                    "uniqueCount": 61,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "E",
+                    "uniqueCount": 192,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "I",
+                    "uniqueCount": 85,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 4,
+            "columnCount": 4,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 4,
+                    "nullCount": 0,
+                    "mean": "2.5",
+                    "stdev": "1.2909944487358056"
+                },
+                {
+                    "fieldPath": "name",
+                    "uniqueCount": 4,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "age",
+                    "uniqueCount": 4,
+                    "nullCount": 0,
+                    "mean": "32.5",
+                    "stdev": "6.454972243679028"
+                },
+                {
+                    "fieldPath": "salary",
+                    "uniqueCount": 4,
+                    "nullCount": 0,
+                    "mean": "65000.0",
+                    "stdev": "12909.944487358056"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 2,
+            "columnCount": 7,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "urn",
+                    "uniqueCount": 1,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "version",
+                    "uniqueCount": 1,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "metadata",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "createdon",
+                    "uniqueCount": 1,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "createdby",
+                    "uniqueCount": 1,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "createdfor",
+                    "uniqueCount": 0,
+                    "nullCount": 2
+                },
+                {
+                    "fieldPath": "aspect",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 3834,
+            "columnCount": 9,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "A",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "B",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "C",
+                    "uniqueCount": 3834,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "D",
+                    "uniqueCount": 76,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "E",
+                    "uniqueCount": 192,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "F",
+                    "uniqueCount": 61,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "G",
+                    "uniqueCount": 40,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "H",
+                    "uniqueCount": 91,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "I",
+                    "uniqueCount": 85,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 5,
+            "columnCount": 6,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "priority",
+                    "uniqueCount": 3,
+                    "nullCount": 1,
+                    "mean": "4.175000011920929",
+                    "stdev": "0.4924429489953036"
+                },
+                {
+                    "fieldPath": "email_address",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "first_name",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "last_name",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "company",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "id",
+                    "uniqueCount": 5,
+                    "nullCount": 0,
+                    "mean": "3.0",
+                    "stdev": "1.5811388300841898"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 10842,
+            "columnCount": 13,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "A",
+                    "uniqueCount": 9661,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "B",
+                    "uniqueCount": 35,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "C",
+                    "uniqueCount": 42,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "D",
+                    "uniqueCount": 6003,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "E",
+                    "uniqueCount": 463,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "F",
+                    "uniqueCount": 23,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "G",
+                    "uniqueCount": 5,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "H",
+                    "uniqueCount": 94,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "I",
+                    "uniqueCount": 8,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "J",
+                    "uniqueCount": 121,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "K",
+                    "uniqueCount": 1379,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "L",
+                    "uniqueCount": 2835,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "M",
+                    "uniqueCount": 35,
+                    "nullCount": 1
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 27,
+            "columnCount": 4,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "DEPARTMENT_NAME",
+                    "uniqueCount": 27,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "MANAGER_ID",
+                    "uniqueCount": 11,
+                    "nullCount": 16,
+                    "mean": "154.9090909090909",
+                    "stdev": "47.139059272443184"
+                },
+                {
+                    "fieldPath": "DEPARTMENT_ID",
+                    "uniqueCount": 27,
+                    "nullCount": 0,
+                    "mean": "140.0",
+                    "stdev": "79.37253933193772"
+                },
+                {
+                    "fieldPath": "LOCATION_ID",
+                    "uniqueCount": 7,
+                    "nullCount": 0,
+                    "mean": "1777.7777777777778",
+                    "stdev": "284.6500325410858"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProfile",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1697353200000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "rowCount": 30000,
+            "columnCount": 9,
+            "fieldProfiles": [
+                {
+                    "fieldPath": "cp_catalog_page_number",
+                    "uniqueCount": 277,
+                    "nullCount": 294,
+                    "mean": "138.8378442065576",
+                    "stdev": "80.01625232480262"
+                },
+                {
+                    "fieldPath": "cp_type",
+                    "uniqueCount": 4,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "cp_description",
+                    "uniqueCount": 29718,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "cp_catalog_number",
+                    "uniqueCount": 109,
+                    "nullCount": 297,
+                    "mean": "54.62360704305962",
+                    "stdev": "31.27039684588463"
+                },
+                {
+                    "fieldPath": "cp_end_date_sk",
+                    "uniqueCount": 97,
+                    "nullCount": 302,
+                    "mean": "2451940.632601522",
+                    "stdev": "634.7367986145963"
+                },
+                {
+                    "fieldPath": "cp_start_date_sk",
+                    "uniqueCount": 91,
+                    "nullCount": 286,
+                    "mean": "2451880.7982769064",
+                    "stdev": "634.1320559175408"
+                },
+                {
+                    "fieldPath": "cp_catalog_page_id",
+                    "uniqueCount": 30000,
+                    "nullCount": 0
+                },
+                {
+                    "fieldPath": "cp_catalog_page_sk",
+                    "uniqueCount": 30000,
+                    "nullCount": 0,
+                    "mean": "15000.5",
+                    "stdev": "8660.398374208891"
+                },
+                {
+                    "fieldPath": "cp_department",
+                    "uniqueCount": 2,
+                    "nullCount": 0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json
new file mode 100644
index 0000000000000..81e9c38d1bbee
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json
@@ -0,0 +1,8002 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "s3",
+            "qualifiedName": "s3",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "mysql",
+            "qualifiedName": "mysql",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "Samples",
+            "qualifiedName": "Samples",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "@admin",
+            "qualifiedName": "@admin",
+            "description": "#  Wikis & Labels\n\n![Gnarly Catalog](https://d33wubrfki0l68.cloudfront.net/c1a54376c45a9276c080f3d10ed25ce61c17bcd2/2b946/img/home/open-source-for-everyone.svg)\n\nYou are reading the wiki for your home space! You can create and edit this information for any source, space, or folder.\n\nThis sidebar always shows the wiki for the current source, space or folder you are browsing.\n\nWhen browsing or previewing datasets, click on the `Open details panel` button to create a wiki or add labels to that dataset.\n\n**Tip:** You can hide the wiki by clicking on the sidebar icon on upper right hand side.",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Space"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "space",
+            "qualifiedName": "space",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Space"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "test_folder",
+            "qualifiedName": "space.test_folder",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+                    "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "dataCharmer",
+            "qualifiedName": "mysql.dataCharmer",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "metagalaxy",
+            "qualifiedName": "mysql.metagalaxy",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "mysql",
+            "qualifiedName": "mysql.mysql",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "northwind",
+            "qualifiedName": "mysql.northwind",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "performance_schema",
+            "qualifiedName": "mysql.performance_schema",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "test_cases",
+            "qualifiedName": "mysql.test_cases",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "samples.dremio.com",
+            "qualifiedName": "Samples.samples.dremio.com",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "Dremio University",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                },
+                {
+                    "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+                    "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "tpcds_sf1000",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                },
+                {
+                    "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+                    "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "catalog_page",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                },
+                {
+                    "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+                    "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+                },
+                {
+                    "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+                    "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                },
+                {
+                    "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+                    "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+                },
+                {
+                    "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+                    "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554"
+                },
+                {
+                    "id": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+                    "urn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"warehouse\"",
+            "name": "warehouse",
+            "qualifiedName": "space.warehouse",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * from Samples.\"samples.dremio.com\".\"NYC-weather.csv\"",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.warehouse",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "I",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "A",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "B",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "C",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "D",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "E",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "F",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "G",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "H",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+                    "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"customers\"",
+            "name": "customers",
+            "qualifiedName": "space.test_folder.customers",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.northwind.customers",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.customers",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "first_name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "last_name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "company",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "priority",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float(24)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "email_address",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+                    "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+                },
+                {
+                    "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+                    "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_aspect\"",
+            "name": "metadata_aspect",
+            "qualifiedName": "space.test_folder.metadata_aspect",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_aspect",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.metadata_aspect",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdby",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdon",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.DateType": {}
+                        }
+                    },
+                    "nativeDataType": "timestamp(23)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "metadata",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "version",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdfor",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "aspect",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+                    "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+                },
+                {
+                    "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+                    "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_index\"",
+            "name": "metadata_index",
+            "qualifiedName": "space.test_folder.metadata_index",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.metadata_index",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "aspect",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "path",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "longVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "stringVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "doubleVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+                    "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+                },
+                {
+                    "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+                    "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_index_view\"",
+            "name": "metadata_index_view",
+            "qualifiedName": "space.test_folder.metadata_index_view",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index_view",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.metadata_index_view",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "doubleVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "path",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+                    "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+                },
+                {
+                    "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+                    "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"orders\"",
+            "name": "orders",
+            "qualifiedName": "space.test_folder.orders",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM mysql.northwind.orders",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.orders",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "customer_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "description",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+                    "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+                },
+                {
+                    "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+                    "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"raw\"",
+            "name": "raw",
+            "qualifiedName": "space.test_folder.raw",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT * FROM s3.warehouse",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "space.test_folder.raw",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "salary",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "age",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Spaces"
+                },
+                {
+                    "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+                    "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+                },
+                {
+                    "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+                    "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"s3\"/\"warehouse\"",
+            "name": "warehouse",
+            "qualifiedName": "s3.warehouse",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "s3.warehouse",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "age",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "salary",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+                    "urn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_aspect\"",
+            "name": "metadata_aspect",
+            "qualifiedName": "mysql.metagalaxy.metadata_aspect",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.metagalaxy.metadata_aspect",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "metadata",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdon",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.DateType": {}
+                        }
+                    },
+                    "nativeDataType": "timestamp(23)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdby",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "createdfor",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "aspect",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "version",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                },
+                {
+                    "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+                    "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_index\"",
+            "name": "metadata_index",
+            "qualifiedName": "mysql.metagalaxy.metadata_index",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.metagalaxy.metadata_index",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "path",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "aspect",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "stringVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "longVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "doubleVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                },
+                {
+                    "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+                    "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_index_view\"",
+            "name": "metadata_index_view",
+            "qualifiedName": "mysql.metagalaxy.metadata_index_view",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.metagalaxy.metadata_index_view",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "urn",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "path",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "doubleVal",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                },
+                {
+                    "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+                    "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"northwind\".\"customers\"",
+            "name": "customers",
+            "qualifiedName": "mysql.northwind.customers",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.northwind.customers",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "priority",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float(24)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "email_address",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "first_name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "company",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "last_name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                },
+                {
+                    "id": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+                    "urn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"mysql\"/\"northwind\".\"orders\"",
+            "name": "orders",
+            "qualifiedName": "mysql.northwind.orders",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "mysql.northwind.orders",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "description",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "customer_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer(32)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+                    "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+                },
+                {
+                    "id": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+                    "urn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"NYC-weather.csv\"",
+            "name": "NYC-weather.csv",
+            "qualifiedName": "Samples.samples.dremio.com.NYC-weather.csv",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.NYC-weather.csv",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "E",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "G",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "H",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "I",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "F",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "A",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "B",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "C",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "D",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                },
+                {
+                    "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+                    "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"googleplaystore.csv\"",
+            "name": "googleplaystore.csv",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:55c3b773b40bb744b4e5946db4e13455"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "F",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "G",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "H",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "I",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "J",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "K",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "L",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "M",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "A",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "B",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "C",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "D",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "E",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                },
+                {
+                    "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+                    "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+                },
+                {
+                    "id": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+                    "urn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"oracle-departments.xlsx\"",
+            "name": "oracle-departments.xlsx",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:55c3b773b40bb744b4e5946db4e13455"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "DEPARTMENT_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "DEPARTMENT_NAME",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "MANAGER_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "LOCATION_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                },
+                {
+                    "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+                    "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+                },
+                {
+                    "id": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+                    "urn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"tpcds_sf1000\".\"catalog_page\".\"1ab266d5-18eb-4780-711d-0fa337fa6c00\".\"0_0_0.parquet\"",
+            "name": "0_0_0.parquet",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "cp_catalog_page_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_page_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_start_date_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_end_date_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_department",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_number",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_page_number",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_description",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_type",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                },
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+                    "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+                },
+                {
+                    "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+                    "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+                },
+                {
+                    "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+                    "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554"
+                },
+                {
+                    "id": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+                    "urn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c"
+                },
+                {
+                    "id": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+                    "urn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),metadata)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdon)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdon)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdby)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdfor)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),aspect)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),aspect)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),version)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),version)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),path)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),path)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),aspect)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),aspect)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),stringVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),longVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),longVal)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),doubleVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),doubleVal)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),path)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),path)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),doubleVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),doubleVal)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),priority)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),priority)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),email_address)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),email_address)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),company)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),company)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),description)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),description)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),salary)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.customers,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.northwind.customers",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.customers,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_aspect,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.metagalaxy.metadata_aspect",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_aspect,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.metagalaxy.metadata_index",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index_view,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.metagalaxy.metadata_index_view",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index_view,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.orders,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM mysql.northwind.orders",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.orders,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+                    "type": "VIEW",
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),id)"
+                    ],
+                    "confidenceScore": 0.9,
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),name)"
+                    ],
+                    "confidenceScore": 0.9,
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),age)"
+                    ],
+                    "confidenceScore": 0.9,
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),salary)"
+                    ],
+                    "confidenceScore": 0.9,
+                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "SELECT\n  *\nFROM s3.warehouse",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1697353200000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),id)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),name)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),age)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),salary)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml
new file mode 100644
index 0000000000000..923dae139681a
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml
@@ -0,0 +1,26 @@
+source:
+  type: dremio
+  config:
+    # Coordinates
+    hostname: localhost
+    port: 9047
+    tls: false
+
+    # Credentials
+    authentication_method: password
+    username: admin
+    password: "2310Admin1234!@"
+
+    platform_instance: test-platform
+
+    include_query_lineage: false
+
+    source_mappings:
+      - platform: s3
+        source_name: samples
+        platform_instance: s3_test_samples
+
+sink:
+  type: file
+  config:
+    filename: "./dremio_mces.json"
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json
new file mode 100644
index 0000000000000..f4dfed45ccf0c
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json
@@ -0,0 +1,2327 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "Samples",
+            "qualifiedName": "Samples",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "samples.dremio.com",
+            "qualifiedName": "Samples.samples.dremio.com",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "Dremio University",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "tpcds_sf1000",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "catalog_page",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+                    "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00",
+            "description": "",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Dremio Folder"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+                    "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+                },
+                {
+                    "id": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+                    "urn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"NYC-weather.csv\"",
+            "name": "NYC-weather.csv",
+            "qualifiedName": "Samples.samples.dremio.com.NYC-weather.csv",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.NYC-weather.csv",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "I",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "H",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "G",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "F",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "E",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "D",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "C",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "B",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "A",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"googleplaystore.csv\"",
+            "name": "googleplaystore.csv",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "E",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "A",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "B",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "C",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "D",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "F",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "G",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "H",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "I",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "J",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "K",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "L",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "M",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+                    "urn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"oracle-departments.xlsx\"",
+            "name": "oracle-departments.xlsx",
+            "qualifiedName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "LOCATION_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "MANAGER_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "DEPARTMENT_NAME",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "DEPARTMENT_ID",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "double(53)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+                    "urn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"tpcds_sf1000\".\"catalog_page\".\"1ab266d5-18eb-4780-711d-0fa337fa6c00\".\"0_0_0.parquet\"",
+            "name": "0_0_0.parquet",
+            "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+            "description": "",
+            "created": {
+                "time": 0
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:dremio"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+            "platform": "urn:li:dataPlatform:dremio",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "cp_start_date_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_page_id",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_page_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_department",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_number",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_end_date_sk",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_catalog_page_number",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "bigint(64)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_description",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "cp_type",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "character varying(65536)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+                    "type": "COPY"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sources"
+                },
+                {
+                    "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+                    "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+                },
+                {
+                    "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+                    "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+                },
+                {
+                    "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+                    "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+                },
+                {
+                    "id": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+                    "urn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+                },
+                {
+                    "id": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+                    "urn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1697353200000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+                    "type": "COPY"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1697353200000,
+        "runId": "dremio-2023_10_15-07_00_00-vftab1",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml
new file mode 100644
index 0000000000000..2b3cea465eebe
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml
@@ -0,0 +1,28 @@
+source:
+  type: dremio
+  config:
+    # Coordinates
+    hostname: localhost
+    port: 9047
+    tls: false
+
+    # Credentials
+    authentication_method: password
+    username: admin
+    password: "2310Admin1234!@"
+
+    include_query_lineage: false
+
+    source_mappings:
+      - platform: s3
+        source_name: samples
+        platform_instance: s3_test_samples
+
+    schema_pattern:
+      allow:
+        - "Samples"
+
+sink:
+  type: file
+  config:
+    filename: "./dremio_mces.json"
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_to_file.yml b/metadata-ingestion/tests/integration/dremio/dremio_to_file.yml
new file mode 100644
index 0000000000000..1ee975722f12d
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_to_file.yml
@@ -0,0 +1,34 @@
+source:
+  type: dremio
+  config:
+    # Coordinates
+    hostname: localhost
+    port: 9047
+    tls: false
+
+    # Credentials
+    authentication_method: password
+    username: admin
+    password: "2310Admin1234!@"
+
+    include_query_lineage: false
+
+    source_mappings:
+      - platform: s3
+        source_name: samples
+        platform_instance: s3_test_samples
+
+    schema_pattern:
+      allow:
+        - ".*"
+
+    profiling:
+      enabled: true
+      include_field_min_value: false
+      include_field_max_value: false
+
+
+sink:
+  type: file
+  config:
+    filename: "./dremio_mces.json"
diff --git a/metadata-ingestion/tests/integration/dremio/setup/mysqld.cnf b/metadata-ingestion/tests/integration/dremio/setup/mysqld.cnf
new file mode 100644
index 0000000000000..4e80d7b70b8f0
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/setup/mysqld.cnf
@@ -0,0 +1,44 @@
+# Copyright (c) 2014, 2021, Oracle and/or its affiliates.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License, version 2.0,
+# as published by the Free Software Foundation.
+#
+# This program is also distributed with certain software (including
+# but not limited to OpenSSL) that is licensed under separate terms,
+# as designated in a particular file or component or in included license
+# documentation.  The authors of MySQL hereby grant you an additional
+# permission to link the program and your derivative works with the
+# separately licensed software that they have included with MySQL.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License, version 2.0, for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+
+#
+# The MySQL  Server configuration file.
+#
+# For explanations see
+# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+
+[mysqld]
+pid-file	= /var/run/mysqld/mysqld.pid
+socket		= /var/run/mysqld/mysqld.sock
+datadir		= /var/lib/mysql
+#log-error	= /var/log/mysql/error.log
+# By default we only accept connections from localhost
+#bind-address	= 127.0.0.1
+bind-address = 0.0.0.0
+# Disabling symbolic-links is recommended to prevent assorted security risks
+symbolic-links=0
+
+server-id         = 223344
+log_bin           = mysql-bin
+binlog_format     = ROW
+binlog_row_image  = FULL
+expire_logs_days  = 10
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dremio/setup/setup.sql b/metadata-ingestion/tests/integration/dremio/setup/setup.sql
new file mode 100644
index 0000000000000..9b8c48bc744f7
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/setup/setup.sql
@@ -0,0 +1,277 @@
+create database metagalaxy;
+use metagalaxy;
+-- create metadata aspect table
+create table metadata_aspect (
+  urn                           varchar(500) not null,
+  aspect                        varchar(200) not null,
+  version                       bigint(20) not null,
+  metadata                      longtext not null,
+  createdon                     datetime(6) not null,
+  createdby                     varchar(255) not null,
+  createdfor                    varchar(255),
+  constraint pk_metadata_aspect primary key (urn,aspect,version)
+);
+
+-- create default records for datahub user
+insert into metadata_aspect (urn, aspect, version, metadata, createdon, createdby) values(
+  'urn:li:corpuser:datahub',
+  'com.linkedin.identity.CorpUserInfo',
+  0,
+  '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}',
+  now(),
+  'urn:li:corpuser:__datahub_system'
+), (
+  'urn:li:corpuser:datahub',
+  'com.linkedin.identity.CorpUserEditableInfo',
+  0,
+  '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/default_avatar.png"}',
+  now(),
+  'urn:li:corpuser:__datahub_system'
+);
+
+-- create metadata index table
+CREATE TABLE metadata_index (
+ `id` BIGINT NOT NULL AUTO_INCREMENT,
+ `urn` VARCHAR(200) NOT NULL COMMENT "This is a column comment about URNs",
+ `aspect` VARCHAR(150) NOT NULL,
+ `path` VARCHAR(150) NOT NULL,
+ `longVal` BIGINT,
+ `stringVal` VARCHAR(200),
+ `doubleVal` DOUBLE,
+ CONSTRAINT id_pk PRIMARY KEY (id),
+ INDEX longIndex (`urn`,`aspect`,`path`,`longVal`),
+ INDEX stringIndex (`urn`,`aspect`,`path`,`stringVal`),
+ INDEX doubleIndex (`urn`,`aspect`,`path`,`doubleVal`)
+) COMMENT="This is a table comment";
+
+-- create view for testing
+CREATE VIEW metadata_index_view AS SELECT id, urn, path, doubleVal FROM metadata_index;
+
+-- -----------------------------------------------------
+-- Some sample data, from https://github.com/dataCharmer/test_db.
+-- -----------------------------------------------------
+
+CREATE SCHEMA IF NOT EXISTS `dataCharmer` ;
+USE `dataCharmer` ;
+
+CREATE TABLE `dataCharmer`.`employees` (
+    emp_no      INT             NOT NULL,
+    birth_date  DATE            NOT NULL,
+    first_name  VARCHAR(14)     NOT NULL,
+    last_name   VARCHAR(16)     NOT NULL,
+    gender      ENUM ('M','F')  NOT NULL,    
+    hire_date   DATE            NOT NULL,
+    PRIMARY KEY (emp_no)
+);
+
+CREATE TABLE `dataCharmer`.`salaries` (
+    emp_no      INT             NOT NULL,
+    salary      INT             NOT NULL,
+    from_date   DATE            NOT NULL,
+    to_date     DATE            NOT NULL,
+    # FOREIGN KEY (emp_no) REFERENCES employees (emp_no) ON DELETE CASCADE,
+    PRIMARY KEY (emp_no, from_date)
+); 
+
+-- now some actual data (10 employees and associated salary data)
+
+INSERT INTO `employees` VALUES (10001,'1953-09-02','Georgi','Facello','M','1986-06-26'),
+(10002,'1964-06-02','Bezalel','Simmel','F','1985-11-21'),
+(10003,'1959-12-03','Parto','Bamford','M','1986-08-28'),
+(10004,'1954-05-01','Chirstian','Koblick','M','1986-12-01'),
+(10005,'1955-01-21','Kyoichi','Maliniak','M','1989-09-12'),
+(10006,'1953-04-20','Anneke','Preusig','F','1989-06-02'),
+(10007,'1957-05-23','Tzvetan','Zielinski','F','1989-02-10'),
+(10008,'1958-02-19','Saniya','Kalloufi','M','1994-09-15'),
+(10009,'1952-04-19','Sumant','Peac','F','1985-02-18'),
+(10010,'1963-06-01','Duangkaew','Piveteau','F','1989-08-24');
+
+INSERT INTO `salaries` VALUES (10001,60117,'1986-06-26','1987-06-26'),
+(10001,62102,'1987-06-26','1988-06-25'),
+(10001,66074,'1988-06-25','1989-06-25'),
+(10001,66596,'1989-06-25','1990-06-25'),
+(10001,66961,'1990-06-25','1991-06-25'),
+(10001,71046,'1991-06-25','1992-06-24'),
+(10001,74333,'1992-06-24','1993-06-24'),
+(10001,75286,'1993-06-24','1994-06-24'),
+(10001,75994,'1994-06-24','1995-06-24'),
+(10001,76884,'1995-06-24','1996-06-23'),
+(10001,80013,'1996-06-23','1997-06-23'),
+(10001,81025,'1997-06-23','1998-06-23'),
+(10001,81097,'1998-06-23','1999-06-23'),
+(10001,84917,'1999-06-23','2000-06-22'),
+(10001,85112,'2000-06-22','2001-06-22'),
+(10001,85097,'2001-06-22','2002-06-22'),
+(10001,88958,'2002-06-22','9999-01-01'),
+(10002,65909,'1996-08-03','1997-08-03'),
+(10002,65909,'1997-08-03','1998-08-03'),
+(10002,67534,'1998-08-03','1999-08-03'),
+(10002,69366,'1999-08-03','2000-08-02'),
+(10002,71963,'2000-08-02','2001-08-02'),
+(10002,72527,'2001-08-02','9999-01-01'),
+(10003,40006,'1995-12-03','1996-12-02'),
+(10003,43616,'1996-12-02','1997-12-02'),
+(10003,43466,'1997-12-02','1998-12-02'),
+(10003,43636,'1998-12-02','1999-12-02'),
+(10003,43478,'1999-12-02','2000-12-01'),
+(10003,43699,'2000-12-01','2001-12-01'),
+(10003,43311,'2001-12-01','9999-01-01'),
+(10004,40054,'1986-12-01','1987-12-01'),
+(10004,42283,'1987-12-01','1988-11-30'),
+(10004,42542,'1988-11-30','1989-11-30'),
+(10004,46065,'1989-11-30','1990-11-30'),
+(10004,48271,'1990-11-30','1991-11-30'),
+(10004,50594,'1991-11-30','1992-11-29'),
+(10004,52119,'1992-11-29','1993-11-29'),
+(10004,54693,'1993-11-29','1994-11-29'),
+(10004,58326,'1994-11-29','1995-11-29'),
+(10004,60770,'1995-11-29','1996-11-28'),
+(10004,62566,'1996-11-28','1997-11-28'),
+(10004,64340,'1997-11-28','1998-11-28'),
+(10004,67096,'1998-11-28','1999-11-28'),
+(10004,69722,'1999-11-28','2000-11-27'),
+(10004,70698,'2000-11-27','2001-11-27'),
+(10004,74057,'2001-11-27','9999-01-01'),
+(10005,78228,'1989-09-12','1990-09-12'),
+(10005,82621,'1990-09-12','1991-09-12'),
+(10005,83735,'1991-09-12','1992-09-11'),
+(10005,85572,'1992-09-11','1993-09-11'),
+(10005,85076,'1993-09-11','1994-09-11'),
+(10005,86050,'1994-09-11','1995-09-11'),
+(10005,88448,'1995-09-11','1996-09-10'),
+(10005,88063,'1996-09-10','1997-09-10'),
+(10005,89724,'1997-09-10','1998-09-10'),
+(10005,90392,'1998-09-10','1999-09-10'),
+(10005,90531,'1999-09-10','2000-09-09'),
+(10005,91453,'2000-09-09','2001-09-09'),
+(10005,94692,'2001-09-09','9999-01-01'),
+(10006,40000,'1990-08-05','1991-08-05'),
+(10006,42085,'1991-08-05','1992-08-04'),
+(10006,42629,'1992-08-04','1993-08-04'),
+(10006,45844,'1993-08-04','1994-08-04'),
+(10006,47518,'1994-08-04','1995-08-04'),
+(10006,47917,'1995-08-04','1996-08-03'),
+(10006,52255,'1996-08-03','1997-08-03'),
+(10006,53747,'1997-08-03','1998-08-03'),
+(10006,56032,'1998-08-03','1999-08-03'),
+(10006,58299,'1999-08-03','2000-08-02'),
+(10006,60098,'2000-08-02','2001-08-02'),
+(10006,59755,'2001-08-02','9999-01-01'),
+(10007,56724,'1989-02-10','1990-02-10'),
+(10007,60740,'1990-02-10','1991-02-10'),
+(10007,62745,'1991-02-10','1992-02-10'),
+(10007,63475,'1992-02-10','1993-02-09'),
+(10007,63208,'1993-02-09','1994-02-09'),
+(10007,64563,'1994-02-09','1995-02-09'),
+(10007,68833,'1995-02-09','1996-02-09'),
+(10007,70220,'1996-02-09','1997-02-08'),
+(10007,73362,'1997-02-08','1998-02-08'),
+(10007,75582,'1998-02-08','1999-02-08'),
+(10007,79513,'1999-02-08','2000-02-08'),
+(10007,80083,'2000-02-08','2001-02-07'),
+(10007,84456,'2001-02-07','2002-02-07'),
+(10007,88070,'2002-02-07','9999-01-01'),
+(10008,46671,'1998-03-11','1999-03-11'),
+(10008,48584,'1999-03-11','2000-03-10'),
+(10008,52668,'2000-03-10','2000-07-31'),
+(10009,60929,'1985-02-18','1986-02-18'),
+(10009,64604,'1986-02-18','1987-02-18'),
+(10009,64780,'1987-02-18','1988-02-18'),
+(10009,66302,'1988-02-18','1989-02-17'),
+(10009,69042,'1989-02-17','1990-02-17'),
+(10009,70889,'1990-02-17','1991-02-17'),
+(10009,71434,'1991-02-17','1992-02-17'),
+(10009,74612,'1992-02-17','1993-02-16'),
+(10009,76518,'1993-02-16','1994-02-16'),
+(10009,78335,'1994-02-16','1995-02-16'),
+(10009,80944,'1995-02-16','1996-02-16'),
+(10009,82507,'1996-02-16','1997-02-15'),
+(10009,85875,'1997-02-15','1998-02-15'),
+(10009,89324,'1998-02-15','1999-02-15'),
+(10009,90668,'1999-02-15','2000-02-15'),
+(10009,93507,'2000-02-15','2001-02-14'),
+(10009,94443,'2001-02-14','2002-02-14'),
+(10009,94409,'2002-02-14','9999-01-01'),
+(10010,72488,'1996-11-24','1997-11-24'),
+(10010,74347,'1997-11-24','1998-11-24'),
+(10010,75405,'1998-11-24','1999-11-24'),
+(10010,78194,'1999-11-24','2000-11-23'),
+(10010,79580,'2000-11-23','2001-11-23'),
+(10010,80324,'2001-11-23','9999-01-01');
+
+-- -----------------------------------------------------
+-- Some sample data, modified from https://github.com/dalers/mywind.
+-- -----------------------------------------------------
+
+SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
+SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
+SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL,ALLOW_INVALID_DATES';
+
+DROP SCHEMA IF EXISTS `northwind` ;
+CREATE SCHEMA IF NOT EXISTS `northwind` DEFAULT CHARACTER SET latin1 ;
+USE `northwind` ;
+
+-- Table `northwind`.`customers`
+CREATE TABLE IF NOT EXISTS `northwind`.`customers` (
+  `id` INT(11) NOT NULL AUTO_INCREMENT,
+  `company` VARCHAR(50) NULL DEFAULT NULL,
+  `last_name` VARCHAR(50) NULL DEFAULT NULL,
+  `first_name` VARCHAR(50) NULL DEFAULT NULL,
+  `email_address` VARCHAR(50) NULL DEFAULT NULL,
+  `priority` FLOAT NULL DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  INDEX `company` (`company` ASC),
+  INDEX `first_name` (`first_name` ASC),
+  INDEX `last_name` (`last_name` ASC))
+ENGINE = InnoDB
+DEFAULT CHARACTER SET = utf8;
+
+-- Table `northwind`.`orders`
+CREATE TABLE IF NOT EXISTS `northwind`.`orders` (
+  `id` INT(11) NOT NULL AUTO_INCREMENT,
+  `description` VARCHAR(50) NULL DEFAULT NULL,
+  `customer_id` INT(11) NOT NULL,
+  PRIMARY KEY (`id`),
+  CONSTRAINT `fk_order_customer`
+    FOREIGN KEY (`customer_id`)
+    REFERENCES `northwind`.`customers`(`id`)
+    ON DELETE NO ACTION
+    ON UPDATE NO ACTION
+)
+ENGINE = InnoDB
+DEFAULT CHARACTER SET = utf8;
+
+-- Now, the actual sample data.
+
+USE `northwind`;
+
+#
+# Dumping data for table 'customers'
+#
+
+INSERT INTO `customers` (`id`, `company`, `last_name`, `first_name`, `email_address`, `priority`) VALUES (1, 'Company A', 'Bedecs', 'Anna', 'Bedecs@xyz.com', 4);
+INSERT INTO `customers` (`id`, `company`, `last_name`, `first_name`, `email_address`, `priority`) VALUES (2, 'Company B', 'Gratacos Solsona', 'Antonio', 'Gratacos@xyz.com', 4.9);
+INSERT INTO `customers` (`id`, `company`, `last_name`, `first_name`, `email_address`, `priority`) VALUES (3, 'Company C', 'Axen', 'Thomas', 'Axen@xyz.com', 4);
+INSERT INTO `customers` (`id`, `company`, `last_name`, `first_name`, `email_address`, `priority`) VALUES (4, 'Company D', 'Lee', 'Christina', 'Lee@xyz.com', 3.8);
+INSERT INTO `customers` (`id`, `company`, `last_name`, `first_name`, `email_address`, `priority`) VALUES (5, 'Company E', 'Donnell', 'Martin', 'Donnell@xyz.com', NULL);
+# 5 records
+
+-- -----------------------------------------------------
+-- Schema for testing different scenarios
+-- -----------------------------------------------------
+
+DROP SCHEMA IF EXISTS `test_cases` ;
+CREATE SCHEMA IF NOT EXISTS `test_cases` DEFAULT CHARACTER SET latin1 ;
+USE `test_cases` ;
+
+-- no data in `test_cases`.`test_empty`
+
+CREATE TABLE IF NOT EXISTS `test_cases`.`test_empty` (
+  `dummy` VARCHAR(50) NULL DEFAULT NULL)
+ENGINE = InnoDB
+DEFAULT CHARACTER SET = utf8;
+
+CREATE TABLE IF NOT EXISTS `test_cases`.`myset` (col SET('a', 'b', 'c', 'd'));
+
+SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;
+SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;
diff --git a/metadata-ingestion/tests/integration/dremio/setup_dremio_admin.sh b/metadata-ingestion/tests/integration/dremio/setup_dremio_admin.sh
new file mode 100755
index 0000000000000..d4b7099ba25cc
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/setup_dremio_admin.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Set variables
+DREMIO_URL="http://localhost:9047"
+ADMIN_USER="admin"
+ADMIN_PASSWORD="2310Admin1234!@"
+ADMIN_FIRST_NAME="Admin"
+ADMIN_LAST_NAME="User"
+ADMIN_EMAIL="admin@dremio.com"
+
+# Wait for Dremio to become available
+until $(curl --output /dev/null --silent --head --fail "$DREMIO_URL"); do
+    echo "Waiting for Dremio to start..."
+    sleep 5
+done
+
+# Create admin user
+echo "Creating Dremio Admin User..."
+RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -X PUT "$DREMIO_URL/apiv2/bootstrap/firstuser" \
+  -H "Content-Type: application/json" \
+  -d "{
+        \"userName\": \"$ADMIN_USER\",
+        \"firstName\": \"$ADMIN_FIRST_NAME\",
+        \"lastName\": \"$ADMIN_LAST_NAME\",
+        \"email\": \"$ADMIN_EMAIL\",
+        \"password\": \"$ADMIN_PASSWORD\"
+      }")
+
+if [ $RESPONSE -eq 200 ]; then
+    echo "Admin user created successfully!"
+else
+    echo "Failed to create admin user. HTTP response: $RESPONSE"
+    exit 1
+fi
diff --git a/metadata-ingestion/tests/integration/dremio/test_data/sample.parquet b/metadata-ingestion/tests/integration/dremio/test_data/sample.parquet
new file mode 100644
index 0000000000000..9e7073d27b720
Binary files /dev/null and b/metadata-ingestion/tests/integration/dremio/test_data/sample.parquet differ
diff --git a/metadata-ingestion/tests/integration/dremio/test_dremio.py b/metadata-ingestion/tests/integration/dremio/test_dremio.py
new file mode 100644
index 0000000000000..401f487d8a14b
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/test_dremio.py
@@ -0,0 +1,593 @@
+import json
+import os
+import subprocess
+from typing import Dict
+
+import boto3
+import pytest
+import requests
+from freezegun import freeze_time
+
+from tests.test_helpers import mce_helpers
+from tests.test_helpers.click_helpers import run_datahub_cmd
+from tests.test_helpers.docker_helpers import wait_for_port
+
+FROZEN_TIME = "2023-10-15 07:00:00"
+MINIO_PORT = 9000
+MYSQL_PORT = 3306
+
+# Dremio server credentials
+DREMIO_HOST = "http://localhost:9047"
+DREMIO_USERNAME = "admin"
+DREMIO_PASSWORD = "2310Admin1234!@"
+MINIO_S3_ENDPOINT = "minio:9000"
+AWS_ACCESS_KEY = "miniouser"
+AWS_SECRET_KEY = "miniopassword"
+AWS_ROOT_PATH = "/warehouse"
+
+
+def is_minio_up(container_name: str) -> bool:
+    """A cheap way to figure out if postgres is responsive on a container"""
+
+    cmd = f"docker logs {container_name} 2>&1 | grep '1 Online'"
+    ret = subprocess.run(
+        cmd,
+        shell=True,
+    )
+    return ret.returncode == 0
+
+
+def is_mysql_up(container_name: str, port: int) -> bool:
+    """A cheap way to figure out if mysql is responsive on a container"""
+
+    cmd = f"docker logs {container_name} 2>&1 | grep '/usr/sbin/mysqld: ready for connections.' | grep {port}"
+    ret = subprocess.run(
+        cmd,
+        shell=True,
+    )
+    return ret.returncode == 0
+
+
+def install_mysql_client(container_name: str) -> None:
+    """
+    This is bit hacky to install mysql-client and connect mysql to start-mysql in container
+    """
+
+    command = f'docker exec --user root {container_name} sh -c  "apt-get update && apt-get install -y mysql-client && /usr/bin/mysql -h test_mysql -u root -prootpwd123"'
+    ret = subprocess.run(command, shell=True, stdout=subprocess.DEVNULL)
+    assert ret.returncode == 0
+
+
+def create_spaces_and_folders(headers):
+    """
+    Create spaces and folders in Dremio
+    """
+    url = f"{DREMIO_HOST}/api/v3/catalog"
+
+    # Create Space
+    payload = {"entityType": "space", "name": "space"}
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to create space: {response.text}"
+
+    # Create Folder inside Space
+    json_data = {"entityType": "folder", "path": ["space", "test_folder"]}
+    response = requests.post(url, headers=headers, data=json.dumps(json_data))
+    assert response.status_code == 200, f"Failed to create folder: {response.text}"
+
+
+def create_sample_source(headers):
+    url = f"{DREMIO_HOST}/api/v3/catalog"
+
+    payload = {
+        "entityType": "source",
+        "config": {
+            "externalBucketList": ["samples.dremio.com"],
+            "credentialType": "NONE",
+            "secure": False,
+            "propertyList": [],
+        },
+        "name": "Samples",
+        "accelerationRefreshPeriod": 3600000,
+        "accelerationGracePeriod": 10800000,
+        "accelerationNeverRefresh": True,
+        "accelerationNeverExpire": True,
+        "accelerationActivePolicyType": "PERIOD",
+        "accelerationRefreshSchedule": "0 0 8 * * *",
+        "accelerationRefreshOnDataChanges": False,
+        "type": "S3",
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to add dataset: {response.text}"
+
+
+def create_s3_source(headers):
+    url = f"{DREMIO_HOST}/api/v3/catalog"
+
+    payload = {
+        "entityType": "source",
+        "name": "s3",
+        "config": {
+            "credentialType": "ACCESS_KEY",
+            "accessKey": AWS_ACCESS_KEY,
+            "accessSecret": AWS_SECRET_KEY,
+            "secure": False,
+            "externalBucketList": ["warehouse"],
+            "enableAsync": True,
+            "enableFileStatusCheck": True,
+            "rootPath": "/",
+            "defaultCtasFormat": "ICEBERG",
+            "propertyList": [
+                {"name": "fs.s3a.access.key", "value": AWS_ACCESS_KEY},
+                {"name": "fs.s3a.secret.key", "value": AWS_SECRET_KEY},
+                {
+                    "name": "fs.s3a.aws.credentials.provider",
+                    "value": "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider",
+                },
+                {"name": "fs.s3a.endpoint", "value": MINIO_S3_ENDPOINT},
+                {"name": "fs.s3a.path.style.access", "value": "True"},
+                {"name": "dremio.s3.compat", "value": "True"},
+                {"name": "fs.s3a.connection.ssl.enabled", "value": "False"},
+            ],
+            "compatibilityMode": True,
+            "whitelistedBuckets": [],
+            "isCachingEnabled": True,
+            "maxCacheSpacePct": 100,
+        },
+        "accelerationRefreshPeriod": 3600000,
+        "accelerationGracePeriod": 10800000,
+        "accelerationActivePolicyType": "PERIOD",
+        "accelerationRefreshSchedule": "0 0 8 * * *",
+        "accelerationRefreshOnDataChanges": False,
+        "metadataPolicy": {
+            "deleteUnavailableDatasets": True,
+            "autoPromoteDatasets": False,
+            "namesRefreshMs": 3600000,
+            "datasetRefreshAfterMs": 3600000,
+            "datasetExpireAfterMs": 10800000,
+            "authTTLMs": 86400000,
+            "datasetUpdateMode": "PREFETCH_QUERIED",
+        },
+        "type": "S3",
+        "accessControlList": {"userControls": [], "roleControls": []},
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to add s3 datasource: {response.text}"
+
+
+def create_mysql_source(headers):
+    url = f"{DREMIO_HOST}/api/v3/catalog"
+
+    payload = {
+        "entityType": "source",
+        "config": {
+            "username": "root",
+            "password": "rootpwd123",
+            "hostname": "test_mysql",
+            "port": MYSQL_PORT,
+            "database": "",
+            "authenticationType": "MASTER",
+            "netWriteTimeout": 60,
+            "fetchSize": 200,
+            "maxIdleConns": 8,
+            "idleTimeSec": 60,
+        },
+        "name": "mysql",
+        "accelerationRefreshPeriod": 3600000,
+        "accelerationGracePeriod": 10800000,
+        "accelerationActivePolicyType": "PERIOD",
+        "accelerationRefreshSchedule": "0 0 8 * * *",
+        "accelerationRefreshOnDataChanges": False,
+        "metadataPolicy": {
+            "deleteUnavailableDatasets": True,
+            "namesRefreshMs": 3600000,
+            "datasetRefreshAfterMs": 3600000,
+            "datasetExpireAfterMs": 10800000,
+            "authTTLMs": 86400000,
+            "datasetUpdateMode": "PREFETCH_QUERIED",
+        },
+        "type": "MYSQL",
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert (
+        response.status_code == 200
+    ), f"Failed to add mysql datasource: {response.text}"
+
+
+def upload_dataset(headers):
+    url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2Fs3%2Fwarehouse"
+    payload = {
+        "entityType": "dataset",
+        "type": "PHYSICAL_DATASET",
+        "path": [
+            "s3",
+            "warehouse",
+        ],
+        "format": {"type": "Parquet"},
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to add dataset: {response.text}"
+
+    url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FNYC-weather.csv"
+
+    payload = {
+        "entityType": "dataset",
+        "type": "PHYSICAL_DATASET",
+        "path": [
+            "Samples",
+            "samples.dremio.com",
+            "NYC-weather.csv",
+        ],
+        "format": {
+            "fieldDelimiter": ",",
+            "quote": '"',
+            "comment": "#",
+            "lineDelimiter": "\r\n",
+            "escape": '"',
+            "extractHeader": False,
+            "trimHeader": True,
+            "skipFirstLine": False,
+            "type": "Text",
+        },
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to add dataset: {response.text}"
+
+    url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FDremio%20University%2Foracle-departments.xlsx"
+
+    payload = {
+        "entityType": "dataset",
+        "type": "PHYSICAL_DATASET",
+        "path": [
+            "Samples",
+            "samples.dremio.com",
+            "Dremio University",
+            "oracle-departments.xlsx",
+        ],
+        "format": {"extractHeader": True, "hasMergedCells": False, "type": "Excel"},
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to add dataset: {response.text}"
+
+    url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FDremio%20University%2Fgoogleplaystore.csv"
+
+    payload = {
+        "entityType": "dataset",
+        "type": "PHYSICAL_DATASET",
+        "path": [
+            "Samples",
+            "samples.dremio.com",
+            "Dremio University",
+            "googleplaystore.csv",
+        ],
+        "format": {
+            "fieldDelimiter": ",",
+            "quote": '"',
+            "comment": "#",
+            "lineDelimiter": "\r\n",
+            "escape": '"',
+            "extractHeader": False,
+            "trimHeader": True,
+            "skipFirstLine": False,
+            "type": "Text",
+        },
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to add dataset: {response.text}"
+
+    url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2Ftpcds_sf1000%2Fcatalog_page%2F1ab266d5-18eb-4780-711d-0fa337fa6c00%2F0_0_0.parquet"
+    payload = {
+        "entityType": "dataset",
+        "type": "PHYSICAL_DATASET",
+        "path": [
+            "Samples",
+            "samples.dremio.com",
+            "tpcds_sf1000",
+            "catalog_page",
+            "1ab266d5-18eb-4780-711d-0fa337fa6c00",
+            "0_0_0.parquet",
+        ],
+        "format": {"type": "Parquet"},
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to add dataset: {response.text}"
+
+
+def create_view(headers):
+    # from s3
+    url = f"{DREMIO_HOST}/api/v3/catalog"
+    payload = {
+        "entityType": "dataset",
+        "type": "VIRTUAL_DATASET",
+        "path": ["space", "test_folder", "raw"],
+        "sql": "SELECT * FROM s3.warehouse",
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to create view: {response.text}"
+
+    url = f"{DREMIO_HOST}/api/v3/catalog"
+    payload = {
+        "entityType": "dataset",
+        "type": "VIRTUAL_DATASET",
+        "path": ["space", "warehouse"],
+        "sql": 'SELECT * from Samples."samples.dremio.com"."NYC-weather.csv"',
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to create view: {response.text}"
+
+    # from mysql
+    payload = {
+        "entityType": "dataset",
+        "type": "VIRTUAL_DATASET",
+        "path": ["space", "test_folder", "customers"],
+        "sql": "SELECT * FROM mysql.northwind.customers",
+        "sqlContext": ["mysql", "northwind"],
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to create view: {response.text}"
+
+    payload = {
+        "entityType": "dataset",
+        "type": "VIRTUAL_DATASET",
+        "path": ["space", "test_folder", "orders"],
+        "sql": "SELECT * FROM mysql.northwind.orders",
+        "sqlContext": ["mysql", "northwind"],
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to create view: {response.text}"
+
+    payload = {
+        "entityType": "dataset",
+        "type": "VIRTUAL_DATASET",
+        "path": ["space", "test_folder", "metadata_aspect"],
+        "sql": "SELECT * FROM mysql.metagalaxy.metadata_aspect",
+        "sqlContext": ["mysql", "metagalaxy"],
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to create view: {response.text}"
+
+    payload = {
+        "entityType": "dataset",
+        "type": "VIRTUAL_DATASET",
+        "path": ["space", "test_folder", "metadata_index"],
+        "sql": "SELECT * FROM mysql.metagalaxy.metadata_index",
+        "sqlContext": ["mysql", "metagalaxy"],
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to create view: {response.text}"
+
+    payload = {
+        "entityType": "dataset",
+        "type": "VIRTUAL_DATASET",
+        "path": ["space", "test_folder", "metadata_index_view"],
+        "sql": "SELECT * FROM mysql.metagalaxy.metadata_index_view",
+        "sqlContext": ["mysql", "metagalaxy"],
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    assert response.status_code == 200, f"Failed to create view: {response.text}"
+
+
+def dremio_header():
+    """
+    Get Dremio authentication token
+    """
+    url = f"{DREMIO_HOST}/apiv2/login"
+    headers = {"Content-Type": "application/json"}
+    payload = {"userName": DREMIO_USERNAME, "password": DREMIO_PASSWORD}
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    response.raise_for_status()  # Raise exception if request failed
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"_dremio{response.json()['token']}",
+    }
+    return headers
+
+
+@pytest.fixture(scope="module")
+def dremio_setup():
+    headers = dremio_header()
+    create_sample_source(headers)
+    create_s3_source(headers)
+    create_mysql_source(headers)
+    create_spaces_and_folders(headers)
+    upload_dataset(headers)
+    create_view(headers)
+
+
+@pytest.fixture(scope="module")
+def test_resources_dir(pytestconfig):
+    return pytestconfig.rootpath / "tests/integration/dremio"
+
+
+@pytest.fixture(scope="module")
+def mock_dremio_service(docker_compose_runner, pytestconfig, test_resources_dir):
+    # Spin up Dremio and MinIO (for mock S3) services using Docker Compose.
+    with docker_compose_runner(
+        test_resources_dir / "docker-compose.yml", "dremio"
+    ) as docker_services:
+        wait_for_port(docker_services, "dremio", 9047, timeout=120)
+        wait_for_port(
+            docker_services,
+            "minio",
+            MINIO_PORT,
+            timeout=120,
+            checker=lambda: is_minio_up("minio"),
+        )
+        wait_for_port(
+            docker_services,
+            "test_mysql",
+            MYSQL_PORT,
+            timeout=120,
+            checker=lambda: is_mysql_up("test_mysql", MYSQL_PORT),
+        )
+
+        # Ensure the admin and data setup scripts have the right permissions
+        subprocess.run(
+            ["chmod", "+x", f"{test_resources_dir}/setup_dremio_admin.sh"], check=True
+        )
+
+        # Run the setup_dremio_admin.sh script
+        admin_setup_cmd = f"{test_resources_dir}/setup_dremio_admin.sh"
+        subprocess.run(admin_setup_cmd, shell=True, check=True)
+
+        install_mysql_client("dremio")
+        yield docker_compose_runner
+
+
+@pytest.fixture(scope="module", autouse=True)
+def s3_bkt(mock_dremio_service):
+    s3 = boto3.resource(
+        "s3",
+        endpoint_url=f"http://localhost:{MINIO_PORT}",
+        aws_access_key_id="miniouser",
+        aws_secret_access_key="miniopassword",
+    )
+    bkt = s3.Bucket("warehouse")
+    bkt.create()
+    return bkt
+
+
+@pytest.fixture(scope="module", autouse=True)
+def populate_minio(pytestconfig, s3_bkt):
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/dremio/test_data/"
+
+    for root, _dirs, files in os.walk(test_resources_dir):
+        for file in files:
+            full_path = os.path.join(root, file)
+            rel_path = os.path.relpath(full_path, test_resources_dir)
+            s3_bkt.upload_file(full_path, rel_path)
+    yield
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_dremio_ingest(
+    test_resources_dir,
+    dremio_setup,
+    pytestconfig,
+    tmp_path,
+):
+    # Run the metadata ingestion pipeline with specific output file
+    config_file = (test_resources_dir / "dremio_to_file.yml").resolve()
+    output_path = tmp_path / "dremio_mces.json"
+
+    run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path)
+
+    # Verify the output
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=output_path,
+        golden_path=test_resources_dir / "dremio_mces_golden.json",
+        ignore_paths=[],
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_dremio_platform_instance_urns(
+    test_resources_dir,
+    dremio_setup,
+    pytestconfig,
+    tmp_path,
+):
+    config_file = (
+        test_resources_dir / "dremio_platform_instance_to_file.yml"
+    ).resolve()
+    output_path = tmp_path / "dremio_mces.json"
+
+    run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path)
+
+    with output_path.open() as f:
+        content = f.read()
+        # Skip if file is empty or just contains brackets
+        if not content or content.strip() in ("[]", "[", "]"):
+            pytest.fail(f"Output file is empty or invalid: {content}")
+
+    try:
+        # Try to load as JSON Lines first
+        mces = []
+        for line in content.splitlines():
+            line = line.strip()
+            if line and line not in ("[", "]"):  # Skip empty lines and bare brackets
+                mce = json.loads(line)
+                mces.append(mce)
+    except json.JSONDecodeError:
+        # If that fails, try loading as a single JSON array
+        try:
+            mces = json.loads(content)
+        except json.JSONDecodeError as e:
+            print(f"Failed to parse file content: {content}")
+            raise e
+
+    # Verify MCEs
+    assert len(mces) > 0, "No MCEs found in output file"
+
+    # Verify the platform instances
+    for mce in mces:
+        if "entityType" not in mce:
+            continue
+
+        # Check dataset URN structure
+        if mce["entityType"] == "dataset" and "entityUrn" in mce:
+            assert (
+                "test-platform.dremio" in mce["entityUrn"]
+            ), f"Platform instance missing in dataset URN: {mce['entityUrn']}"
+
+        # Check aspects for both datasets and containers
+        if "aspectName" in mce:
+            # Check dataPlatformInstance aspect
+            if mce["aspectName"] == "dataPlatformInstance":
+                aspect = mce["aspect"]
+                if not isinstance(aspect, Dict) or "json" not in aspect:
+                    continue
+
+                aspect_json = aspect["json"]
+                if not isinstance(aspect_json, Dict):
+                    continue
+
+                if "instance" not in aspect_json:
+                    continue
+
+                instance = aspect_json["instance"]
+                expected_instance = "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+                assert (
+                    instance == expected_instance
+                ), f"Invalid platform instance format: {instance}"
+
+    # Verify against golden file
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=output_path,
+        golden_path=test_resources_dir / "dremio_platform_instance_mces_golden.json",
+        ignore_paths=[],
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_dremio_schema_filter(
+    test_resources_dir,
+    dremio_setup,
+    pytestconfig,
+    tmp_path,
+):
+    config_file = (test_resources_dir / "dremio_schema_filter_to_file.yml").resolve()
+    output_path = tmp_path / "dremio_mces.json"
+
+    run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path)
+
+    # Verify against golden file
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=output_path,
+        golden_path=test_resources_dir / "dremio_schema_filter_mces_golden.json",
+        ignore_paths=[],
+    )
diff --git a/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json b/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json
index 1b91925289845..a4fd9843c5cf4 100644
--- a/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json
+++ b/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json
@@ -9,8 +9,33 @@
                         "removed": false
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:deprecated"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpGroup:MOCK_OWNER",
+                                "type": "BUSINESS_OWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": {
+                        "customProperties": {},
                         "description": "Driver ID",
                         "dataType": "ORDINAL",
                         "sources": [
@@ -23,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -36,8 +62,18 @@
                         "removed": false
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:needs_documentation"
+                            }
+                        ]
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": {
+                        "customProperties": {},
                         "description": "Conv rate",
                         "dataType": "CONTINUOUS",
                         "sources": [
@@ -50,7 +86,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -65,6 +102,7 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": {
+                        "customProperties": {},
                         "description": "Acc rate",
                         "dataType": "CONTINUOUS",
                         "sources": [
@@ -77,7 +115,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -92,6 +131,7 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": {
+                        "customProperties": {},
                         "description": "Avg daily trips",
                         "dataType": "ORDINAL",
                         "sources": [
@@ -104,7 +144,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -119,6 +160,7 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": {
+                        "customProperties": {},
                         "description": "String feature",
                         "dataType": "TEXT",
                         "sources": [
@@ -131,7 +173,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -151,6 +194,30 @@
                         "removed": false
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:deprecated"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpGroup:MOCK_OWNER",
+                                "type": "BUSINESS_OWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": {
                         "customProperties": {},
@@ -170,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -189,7 +257,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -204,6 +273,7 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": {
+                        "customProperties": {},
                         "dataType": "CONTINUOUS",
                         "sources": [
                             "urn:li:dataset:(urn:li:dataPlatform:request,vals_to_add,PROD)",
@@ -216,7 +286,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -231,6 +302,7 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": {
+                        "customProperties": {},
                         "dataType": "CONTINUOUS",
                         "sources": [
                             "urn:li:dataset:(urn:li:dataPlatform:request,vals_to_add,PROD)",
@@ -243,7 +315,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -278,7 +351,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -297,7 +371,40 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "feast-repository-test"
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:deprecated",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "deprecated"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:needs_documentation",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "needs_documentation"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "feast-repository-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/feast/feature_store/data/registry.db b/metadata-ingestion/tests/integration/feast/feature_store/data/registry.db
index a511ff56c9770..5dca29d92afe5 100644
Binary files a/metadata-ingestion/tests/integration/feast/feature_store/data/registry.db and b/metadata-ingestion/tests/integration/feast/feature_store/data/registry.db differ
diff --git a/metadata-ingestion/tests/integration/feast/feature_store/features.py b/metadata-ingestion/tests/integration/feast/feature_store/features.py
index a6e6cd3616e92..dcfd417637958 100644
--- a/metadata-ingestion/tests/integration/feast/feature_store/features.py
+++ b/metadata-ingestion/tests/integration/feast/feature_store/features.py
@@ -19,6 +19,8 @@
     join_keys=["driver_id"],
     value_type=ValueType.INT64,
     description="Driver ID",
+    owner="MOCK_OWNER",
+    tags={"name": "deprecated"},
 )
 
 driver_hourly_stats_view = FeatureView(
@@ -29,7 +31,7 @@
         Field(
             name="conv_rate",
             dtype=feast.types.Float64,
-            tags=dict(description="Conv rate"),
+            tags={"name": "needs_documentation", "description": "Conv rate"},
         ),
         Field(
             name="acc_rate",
@@ -49,7 +51,8 @@
     ],
     online=True,
     source=driver_hourly_stats_source,
-    tags={},
+    tags={"name": "deprecated"},
+    owner="MOCK_OWNER",
 )
 
 input_request = RequestSource(
diff --git a/metadata-ingestion/tests/integration/feast/test_feast_repository.py b/metadata-ingestion/tests/integration/feast/test_feast_repository.py
index a6bdce6722289..7f04337145dc3 100644
--- a/metadata-ingestion/tests/integration/feast/test_feast_repository.py
+++ b/metadata-ingestion/tests/integration/feast/test_feast_repository.py
@@ -19,6 +19,15 @@ def test_feast_repository_ingest(pytestconfig, tmp_path, mock_time):
                 "config": {
                     "path": str(test_resources_dir / "feature_store"),
                     "environment": "PROD",
+                    "enable_tag_extraction": True,
+                    "enable_owner_extraction": True,
+                    "owner_mappings": [
+                        {
+                            "feast_owner_name": "MOCK_OWNER",
+                            "datahub_owner_urn": "urn:li:corpGroup:MOCK_OWNER",
+                            "datahub_ownership_type": "BUSINESS_OWNER",
+                        }
+                    ],
                 },
             },
             "sink": {
diff --git a/metadata-ingestion/tests/integration/file/metadata_file.json b/metadata-ingestion/tests/integration/file/metadata_file.json
new file mode 100644
index 0000000000000..27d49e3661473
--- /dev/null
+++ b/metadata-ingestion/tests/integration/file/metadata_file.json
@@ -0,0 +1,78 @@
+[
+    {
+        "entityType": "dataset",
+        "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+        "changeType": "UPSERT",
+        "aspectName": "datasetProperties",
+        "aspect": {
+            "json": {
+                "name": "TABLE_1",
+                "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
+                "description": "Comment for Table",
+                "tags": []
+            }
+        },
+        "systemMetadata": {
+            "lastObserved": 1615443388097,
+            "runId": "snowflake-2023_12_18-10_16_09",
+            "lastRunId": "no-run-id-provided"
+        }
+    },
+    {
+        "entityType": "dataset",
+        "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+        "changeType": "UPSERT",
+        "aspectName": "datasetProperties",
+        "aspect": {
+            "json": {
+                "name": "TABLE_2",
+                "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
+                "description": "Comment for Table",
+                "tags": []
+            }
+        },
+        "systemMetadata": {
+            "lastObserved": 1615443388097,
+            "runId": "snowflake-2023_12_18-10_16_09",
+            "lastRunId": "no-run-id-provided"
+        }
+    },
+    {
+        "entityType": "dataset",
+        "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+        "changeType": "UPSERT",
+        "aspectName": "datasetProperties",
+        "aspect": {
+            "json": {
+                "name": "TABLE_3",
+                "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
+                "description": "Comment for Table",
+                "tags": []
+            }
+        },
+        "systemMetadata": {
+            "lastObserved": 1615443388097,
+            "runId": "snowflake-2023_12_18-10_16_09",
+            "lastRunId": "no-run-id-provided"
+        }
+    },
+    {
+        "entityType": "dataset",
+        "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+        "changeType": "UPSERT",
+        "aspectName": "datasetProperties",
+        "aspect": {
+            "json": {
+                "name": "TABLE_4",
+                "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
+                "description": "Comment for Table",
+                "tags": []
+            }
+        },
+        "systemMetadata": {
+            "lastObserved": 1615443388097,
+            "runId": "snowflake-2023_12_18-10_16_09",
+            "lastRunId": "no-run-id-provided"
+        }
+    }
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/file/state_golden.json b/metadata-ingestion/tests/integration/file/state_golden.json
new file mode 100644
index 0000000000000..58f222110dc88
--- /dev/null
+++ b/metadata-ingestion/tests/integration/file/state_golden.json
@@ -0,0 +1,26 @@
+[
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(file,dummy_stateful,prod),default_stale_entity_removal)",
+    "changeType": "UPSERT",
+    "aspectName": "datahubIngestionCheckpoint",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1730898142020,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "pipelineName": "dummy_stateful",
+            "platformInstanceId": "",
+            "config": "",
+            "state": {
+                "formatVersion": "1.0",
+                "serde": "utf-8",
+                "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)\"]}"
+            },
+            "runId": "test-run"
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/file/test_file_source.py b/metadata-ingestion/tests/integration/file/test_file_source.py
new file mode 100644
index 0000000000000..fb2cfd9fc9f0f
--- /dev/null
+++ b/metadata-ingestion/tests/integration/file/test_file_source.py
@@ -0,0 +1,49 @@
+import pathlib
+from unittest import mock
+
+from datahub.ingestion.run.pipeline import Pipeline
+from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState
+from tests.test_helpers import mce_helpers
+
+
+def test_stateful_ingestion(tmp_path, pytestconfig):
+    state_file_name = "state.json"
+    golden_state_file_name = "state_golden.json"
+
+    test_resources_dir: pathlib.Path = pytestconfig.rootpath / "tests/integration/file"
+    pipeline_config = {
+        "run_id": "test-run",
+        "pipeline_name": "dummy_stateful",
+        "source": {
+            "type": "file",
+            "config": {
+                "filename": str(test_resources_dir / "metadata_file.json"),
+                "stateful_ingestion": {
+                    "enabled": True,
+                    "remove_stale_metadata": True,
+                    "state_provider": {
+                        "type": "file",
+                        "config": {
+                            "filename": f"{tmp_path}/{state_file_name}",
+                        },
+                    },
+                },
+            },
+        },
+        "sink": {
+            "type": "blackhole",
+            "config": {},
+        },
+    }
+    with mock.patch(
+        "datahub.ingestion.source.state.stale_entity_removal_handler.StaleEntityRemovalHandler._get_state_obj"
+    ) as mock_state:
+        mock_state.return_value = GenericCheckpointState(serde="utf-8")
+        pipeline = Pipeline.create(pipeline_config)
+        pipeline.run()
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=tmp_path / state_file_name,
+        golden_path=f"{test_resources_dir}/{golden_state_file_name}",
+    )
diff --git a/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json b/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json
index 29b186978a76a..7934153051c60 100644
--- a/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json
+++ b/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json
@@ -17,6 +17,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataFlow",
     "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
@@ -62,9 +78,17 @@
     "aspect": {
         "json": {
             "customProperties": {
+                "connector_id": "calendar_elected",
+                "connector_type": "postgres",
                 "paused": "False",
                 "sync_frequency": "1440",
-                "destination_id": "'interval_unconstitutional'"
+                "destination_id": "interval_unconstitutional",
+                "source.platform": "postgres",
+                "source.env": "DEV",
+                "source.database": "postgres_db",
+                "destination.platform": "snowflake",
+                "destination.env": "PROD",
+                "destination.database": "test_database"
             },
             "name": "postgres",
             "type": {
@@ -79,6 +103,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
@@ -149,38 +189,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
@@ -304,8 +312,8 @@
         "json": {
             "timestampMillis": 1695191853000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "STARTED"
         }
@@ -325,8 +333,8 @@
         "json": {
             "timestampMillis": 1695191885000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "COMPLETE",
             "result": {
@@ -427,8 +435,8 @@
         "json": {
             "timestampMillis": 1696343730000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "STARTED"
         }
@@ -448,8 +456,8 @@
         "json": {
             "timestampMillis": 1696343732000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "COMPLETE",
             "result": {
@@ -550,8 +558,8 @@
         "json": {
             "timestampMillis": 1696343755000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "STARTED"
         }
@@ -571,8 +579,8 @@
         "json": {
             "timestampMillis": 1696343790000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "COMPLETE",
             "result": {
@@ -587,38 +595,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataProcessInstance",
     "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a",
diff --git a/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json b/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json
index 0cd3bb83f90f5..c46bd8fb65d87 100644
--- a/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json
+++ b/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json
@@ -17,6 +17,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataFlow",
     "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
@@ -62,9 +78,17 @@
     "aspect": {
         "json": {
             "customProperties": {
+                "connector_id": "calendar_elected",
+                "connector_type": "postgres",
                 "paused": "False",
                 "sync_frequency": "1440",
-                "destination_id": "'interval_unconstitutional'"
+                "destination_id": "interval_unconstitutional",
+                "source.platform": "postgres",
+                "source.env": "DEV",
+                "source.database": "postgres_db",
+                "destination.platform": "snowflake",
+                "destination.env": "PROD",
+                "destination.database": "test_database"
             },
             "name": "postgres",
             "type": {
@@ -79,6 +103,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
@@ -149,38 +189,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
@@ -312,8 +320,8 @@
         "json": {
             "timestampMillis": 1695191853000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "STARTED"
         }
@@ -333,8 +341,8 @@
         "json": {
             "timestampMillis": 1695191885000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "COMPLETE",
             "result": {
@@ -435,8 +443,8 @@
         "json": {
             "timestampMillis": 1696343730000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "STARTED"
         }
@@ -456,8 +464,8 @@
         "json": {
             "timestampMillis": 1696343732000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "COMPLETE",
             "result": {
@@ -558,8 +566,8 @@
         "json": {
             "timestampMillis": 1696343755000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "STARTED"
         }
@@ -579,8 +587,8 @@
         "json": {
             "timestampMillis": 1696343790000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "status": "COMPLETE",
             "result": {
@@ -595,38 +603,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataProcessInstance",
     "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a",
diff --git a/metadata-ingestion/tests/integration/fivetran/test_fivetran.py b/metadata-ingestion/tests/integration/fivetran/test_fivetran.py
index 0f5d098ee39c4..2e6c2b1370d16 100644
--- a/metadata-ingestion/tests/integration/fivetran/test_fivetran.py
+++ b/metadata-ingestion/tests/integration/fivetran/test_fivetran.py
@@ -13,6 +13,7 @@
 from datahub.ingestion.source.fivetran.config import (
     BigQueryDestinationConfig,
     FivetranSourceConfig,
+    PlatformDetail,
     SnowflakeDestinationConfig,
 )
 from datahub.ingestion.source.fivetran.fivetran import FivetranSource
@@ -43,7 +44,9 @@ def default_query_results(
         return []
     elif query == fivetran_log_query.get_connectors_query():
         return connector_query_results
-    elif query == fivetran_log_query.get_table_lineage_query():
+    elif query == fivetran_log_query.get_table_lineage_query(
+        connector_ids=["calendar_elected"]
+    ):
         return [
             {
                 "connector_id": "calendar_elected",
@@ -64,7 +67,9 @@ def default_query_results(
                 "destination_schema_name": "postgres_public",
             },
         ]
-    elif query == fivetran_log_query.get_column_lineage_query():
+    elif query == fivetran_log_query.get_column_lineage_query(
+        connector_ids=["calendar_elected"]
+    ):
         return [
             {
                 "source_table_id": "10040",
@@ -100,65 +105,31 @@ def default_query_results(
                 "email": "abc.xyz@email.com",
             }
         ]
-    elif query == fivetran_log_query.get_sync_logs_query().format(
-        db_clause=fivetran_log_query.db_clause, syncs_interval=7
+    elif query == fivetran_log_query.get_sync_logs_query(
+        syncs_interval=7,
+        connector_ids=["calendar_elected"],
     ):
         return [
             {
                 "connector_id": "calendar_elected",
                 "sync_id": "4c9a03d6-eded-4422-a46a-163266e58243",
-                "message_event": "sync_start",
-                "message_data": None,
-                "time_stamp": datetime.datetime(2023, 9, 20, 6, 37, 32, 606000),
-            },
-            {
-                "connector_id": "calendar_elected",
-                "sync_id": "f773d1e9-c791-48f4-894f-8cf9b3dfc834",
-                "message_event": "sync_start",
-                "message_data": None,
-                "time_stamp": datetime.datetime(2023, 10, 3, 14, 35, 30, 345000),
-            },
-            {
-                "connector_id": "calendar_elected",
-                "sync_id": "63c2fc85-600b-455f-9ba0-f576522465be",
-                "message_event": "sync_start",
-                "message_data": None,
-                "time_stamp": datetime.datetime(2023, 10, 3, 14, 35, 55, 401000),
-            },
-            {
-                "connector_id": "calendar_elected",
-                "sync_id": "e773e1e9-c791-46f4-894f-8ch9b3dfc832",
-                "message_event": "sync_start",
-                "message_data": None,
-                "time_stamp": datetime.datetime(2023, 10, 3, 14, 37, 5, 403000),
-            },
-            {
-                "connector_id": "calendar_elected",
-                "sync_id": "4c9a03d6-eded-4422-a46a-163266e58243",
-                "message_event": "sync_end",
-                "message_data": '"{\\"status\\":\\"SUCCESSFUL\\"}"',
-                "time_stamp": datetime.datetime(2023, 9, 20, 6, 38, 5, 56000),
+                "start_time": datetime.datetime(2023, 9, 20, 6, 37, 32, 606000),
+                "end_time": datetime.datetime(2023, 9, 20, 6, 38, 5, 56000),
+                "end_message_data": '"{\\"status\\":\\"SUCCESSFUL\\"}"',
             },
             {
                 "connector_id": "calendar_elected",
                 "sync_id": "f773d1e9-c791-48f4-894f-8cf9b3dfc834",
-                "message_event": "sync_end",
-                "message_data": '"{\\"reason\\":\\"Sync has been cancelled because of a user action in the dashboard.Standard Config updated.\\",\\"status\\":\\"CANCELED\\"}"',
-                "time_stamp": datetime.datetime(2023, 10, 3, 14, 35, 31, 512000),
+                "start_time": datetime.datetime(2023, 10, 3, 14, 35, 30, 345000),
+                "end_time": datetime.datetime(2023, 10, 3, 14, 35, 31, 512000),
+                "end_message_data": '"{\\"reason\\":\\"Sync has been cancelled because of a user action in the dashboard.Standard Config updated.\\",\\"status\\":\\"CANCELED\\"}"',
             },
             {
                 "connector_id": "calendar_elected",
                 "sync_id": "63c2fc85-600b-455f-9ba0-f576522465be",
-                "message_event": "sync_end",
-                "message_data": '"{\\"reason\\":\\"java.lang.RuntimeException: FATAL: too many connections for role \\\\\\"hxwraqld\\\\\\"\\",\\"taskType\\":\\"reconnect\\",\\"status\\":\\"FAILURE_WITH_TASK\\"}"',
-                "time_stamp": datetime.datetime(2023, 10, 3, 14, 36, 29, 678000),
-            },
-            {
-                "connector_id": "calendar_elected",
-                "sync_id": "e773e1e9-c791-46f4-894f-8ch9b3dfc832",
-                "message_event": "sync_end",
-                "message_data": None,
-                "time_stamp": datetime.datetime(2023, 10, 3, 14, 37, 35, 478000),
+                "start_time": datetime.datetime(2023, 10, 3, 14, 35, 55, 401000),
+                "end_time": datetime.datetime(2023, 10, 3, 14, 36, 29, 678000),
+                "end_message_data": '"{\\"reason\\":\\"java.lang.RuntimeException: FATAL: too many connections for role \\\\\\"hxwraqld\\\\\\"\\",\\"taskType\\":\\"reconnect\\",\\"status\\":\\"FAILURE_WITH_TASK\\"}"',
             },
         ]
     # Unreachable code
@@ -210,11 +181,9 @@ def test_fivetran_with_snowflake_dest(pytestconfig, tmp_path):
                                 "interval_unconstitutional",
                             ]
                         },
-                        "sources_to_database": {
-                            "calendar_elected": "postgres_db",
-                        },
                         "sources_to_platform_instance": {
                             "calendar_elected": {
+                                "database": "postgres_db",
                                 "env": "DEV",
                             }
                         },
@@ -301,12 +270,11 @@ def test_fivetran_with_snowflake_dest_and_null_connector_user(pytestconfig, tmp_
                                 "interval_unconstitutional",
                             ]
                         },
-                        "sources_to_database": {
-                            "calendar_elected": "postgres_db",
-                        },
                         "sources_to_platform_instance": {
                             "calendar_elected": {
+                                "platform": "postgres",
                                 "env": "DEV",
+                                "database": "postgres_db",
                             }
                         },
                     },
@@ -404,3 +372,34 @@ def test_rename_destination_config():
         match="destination_config is deprecated, please use snowflake_destination_config instead.",
     ):
         FivetranSourceConfig.parse_obj(config_dict)
+
+
+def test_compat_sources_to_database() -> None:
+    config_dict = {
+        # We just need a valid fivetran_log_config to test the compat transformation.
+        "fivetran_log_config": {
+            "destination_platform": "snowflake",
+            "snowflake_destination_config": {
+                "account_id": "testid",
+                "warehouse": "test_wh",
+                "username": "test",
+                "password": "test@123",
+                "database": "test_database",
+                "role": "testrole",
+                "log_schema": "test",
+            },
+        },
+        "sources_to_database": {"calendar_elected": "my_db", "connector_2": "my_db_2"},
+        "sources_to_platform_instance": {"calendar_elected": {"env": "DEV"}},
+    }
+
+    with pytest.warns(
+        ConfigurationWarning,
+        match=r"sources_to_database.*deprecated",
+    ):
+        config = FivetranSourceConfig.parse_obj(config_dict)
+
+    assert config.sources_to_platform_instance == {
+        "calendar_elected": PlatformDetail(env="DEV", database="my_db"),
+        "connector_2": PlatformDetail(database="my_db_2"),
+    }
diff --git a/metadata-ingestion/tests/integration/git/test_git_clone.py b/metadata-ingestion/tests/integration/git/test_git_clone.py
index 773e84cbf7488..01e075930998a 100644
--- a/metadata-ingestion/tests/integration/git/test_git_clone.py
+++ b/metadata-ingestion/tests/integration/git/test_git_clone.py
@@ -1,17 +1,19 @@
-import doctest
 import os
+import pathlib
 
 import pytest
 from pydantic import SecretStr
 
+import datahub.ingestion.source.git.git_import
 from datahub.configuration.common import ConfigurationWarning
 from datahub.configuration.git import GitInfo, GitReference
 from datahub.ingestion.source.git.git_import import GitClone
+from datahub.testing.doctest import assert_doctest
 
 LOOKML_TEST_SSH_KEY = os.environ.get("DATAHUB_LOOKML_GIT_TEST_SSH_KEY")
 
 
-def test_base_url_guessing():
+def test_base_url_guessing() -> None:
     # Basic GitHub repo.
     config = GitInfo(repo="https://github.com/datahub-project/datahub", branch="master")
     assert config.repo_ssh_locator == "git@github.com:datahub-project/datahub.git"
@@ -69,7 +71,7 @@ def test_base_url_guessing():
         )
 
 
-def test_github_branch():
+def test_github_branch() -> None:
     config = GitInfo(
         repo="owner/repo",
     )
@@ -82,18 +84,37 @@ def test_github_branch():
     assert config.branch_for_clone == "main"
 
 
-def test_sanitize_repo_url():
-    import datahub.ingestion.source.git.git_import
+def test_url_subdir() -> None:
+    git_ref = GitReference(repo="https://github.com/org/repo", url_subdir="dbt")
+    assert (
+        git_ref.get_url_for_file_path("model.sql")
+        == "https://github.com/org/repo/blob/main/dbt/model.sql"
+    )
+
+    git_ref = GitReference(repo="https://gitlab.com/org/repo", url_subdir="dbt")
+    assert (
+        git_ref.get_url_for_file_path("model.sql")
+        == "https://gitlab.com/org/repo/-/blob/main/dbt/model.sql"
+    )
 
+    git_ref = GitReference(repo="https://github.com/org/repo", url_subdir="")
     assert (
-        doctest.testmod(
-            datahub.ingestion.source.git.git_import, raise_on_error=True
-        ).attempted
-        == 3
+        git_ref.get_url_for_file_path("model.sql")
+        == "https://github.com/org/repo/blob/main/model.sql"
     )
 
+    git_ref = GitReference(repo="https://github.com/org/repo", url_subdir="dbt/models")
+    assert (
+        git_ref.get_url_for_file_path("model.sql")
+        == "https://github.com/org/repo/blob/main/dbt/models/model.sql"
+    )
+
+
+def test_sanitize_repo_url() -> None:
+    assert_doctest(datahub.ingestion.source.git.git_import)
+
 
-def test_git_clone_public(tmp_path):
+def test_git_clone_public(tmp_path: pathlib.Path) -> None:
     git_clone = GitClone(str(tmp_path))
     checkout_dir = git_clone.clone(
         ssh_key=None,
@@ -113,7 +134,7 @@ def test_git_clone_public(tmp_path):
     LOOKML_TEST_SSH_KEY is None,
     reason="DATAHUB_LOOKML_GIT_TEST_SSH_KEY env variable is not configured",
 )
-def test_git_clone_private(tmp_path):
+def test_git_clone_private(tmp_path: pathlib.Path) -> None:
     git_clone = GitClone(str(tmp_path))
     secret_key = SecretStr(LOOKML_TEST_SSH_KEY) if LOOKML_TEST_SSH_KEY else None
 
diff --git a/metadata-ingestion/tests/integration/iceberg/docker-compose.yml b/metadata-ingestion/tests/integration/iceberg/docker-compose.yml
index 8baae6e8ab636..8a05ac7481fe2 100644
--- a/metadata-ingestion/tests/integration/iceberg/docker-compose.yml
+++ b/metadata-ingestion/tests/integration/iceberg/docker-compose.yml
@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   spark-iceberg:
     image: tabulario/spark-iceberg:3.3.2_1.3.0
diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json
index 3321fcac0d73e..4b2afb29ddda8 100644
--- a/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json
+++ b/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json
@@ -11,6 +11,7 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "name": "another_taxis",
                         "customProperties": {
                             "owner": "root",
                             "created-at": "2024-06-27T17:29:32.492204247Z",
diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json
index b017b6cd31520..477f719ef9317 100644
--- a/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json
+++ b/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json
@@ -11,6 +11,7 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "name": "taxis",
                         "customProperties": {
                             "owner": "root",
                             "created-at": "2024-05-22T14:08:04.001538500Z",
diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_multiprocessing_to_file.yml b/metadata-ingestion/tests/integration/iceberg/iceberg_multiprocessing_to_file.yml
new file mode 100644
index 0000000000000..e5e866fb561c9
--- /dev/null
+++ b/metadata-ingestion/tests/integration/iceberg/iceberg_multiprocessing_to_file.yml
@@ -0,0 +1,22 @@
+run_id: iceberg-test
+
+source:
+  type: iceberg
+  config:
+    processing_threads: 5
+    catalog:
+      default:
+        type: rest
+        uri: http://localhost:8181
+        s3.access-key-id: admin
+        s3.secret-access-key: password
+        s3.region: us-east-1
+        warehouse: s3a://warehouse/wh/
+        s3.endpoint: http://localhost:9000
+    user_ownership_property: owner
+    group_ownership_property: owner
+
+sink:
+  type: file
+  config:
+    filename: "./iceberg_mces.json"
diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json
index 453a79494fa25..6d2ca013d81d0 100644
--- a/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json
+++ b/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json
@@ -11,6 +11,7 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "name": "taxis",
                         "customProperties": {
                             "owner": "root",
                             "created-at": "2024-05-22T14:10:22.926080700Z",
diff --git a/metadata-ingestion/tests/integration/iceberg/test_iceberg.py b/metadata-ingestion/tests/integration/iceberg/test_iceberg.py
index 5a12afa457f01..85809e557dd8d 100644
--- a/metadata-ingestion/tests/integration/iceberg/test_iceberg.py
+++ b/metadata-ingestion/tests/integration/iceberg/test_iceberg.py
@@ -1,5 +1,5 @@
 import subprocess
-from typing import Any, Dict, List
+from typing import Any, Dict
 from unittest.mock import patch
 
 import pytest
@@ -18,6 +18,13 @@
 FROZEN_TIME = "2020-04-14 07:00:00"
 GMS_PORT = 8080
 GMS_SERVER = f"http://localhost:{GMS_PORT}"
+# These paths change from one instance run of the clickhouse docker to the other, and the FROZEN_TIME does not apply to
+# these.
+PATHS_IN_GOLDEN_FILE_TO_IGNORE = [
+    r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['created-at'\]",
+    r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['snapshot-id'\]",
+    r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['manifest-list'\]",
+]
 
 
 @pytest.fixture(autouse=True, scope="module")
@@ -35,6 +42,36 @@ def spark_submit(file_path: str, args: str = "") -> None:
     assert ret.returncode == 0
 
 
+@freeze_time(FROZEN_TIME)
+def test_multiprocessing_iceberg_ingest(
+    docker_compose_runner, pytestconfig, tmp_path, mock_time
+):
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg/"
+
+    with docker_compose_runner(
+        test_resources_dir / "docker-compose.yml", "iceberg"
+    ) as docker_services:
+        wait_for_port(docker_services, "spark-iceberg", 8888, timeout=120)
+
+        # Run the create.py pyspark file to populate the table.
+        spark_submit("/home/iceberg/setup/create.py", "nyc.taxis")
+
+        # Run the metadata ingestion pipeline.
+        config_file = (
+            test_resources_dir / "iceberg_multiprocessing_to_file.yml"
+        ).resolve()
+        run_datahub_cmd(
+            ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
+        )
+        # Verify the output.
+        mce_helpers.check_golden_file(
+            pytestconfig,
+            ignore_paths=PATHS_IN_GOLDEN_FILE_TO_IGNORE,
+            output_path=tmp_path / "iceberg_mces.json",
+            golden_path=test_resources_dir / "iceberg_ingest_mces_golden.json",
+        )
+
+
 @freeze_time(FROZEN_TIME)
 def test_iceberg_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg/"
@@ -52,16 +89,10 @@ def test_iceberg_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time
         run_datahub_cmd(
             ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
         )
-        # These paths change from one instance run of the clickhouse docker to the other, and the FROZEN_TIME does not apply to these.
-        ignore_paths: List[str] = [
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['created-at'\]",
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['snapshot-id'\]",
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['manifest-list'\]",
-        ]
         # Verify the output.
         mce_helpers.check_golden_file(
             pytestconfig,
-            ignore_paths=ignore_paths,
+            ignore_paths=PATHS_IN_GOLDEN_FILE_TO_IGNORE,
             output_path=tmp_path / "iceberg_mces.json",
             golden_path=test_resources_dir / "iceberg_ingest_mces_golden.json",
         )
@@ -170,16 +201,10 @@ def test_iceberg_stateful_ingest(
             pipeline=pipeline_run2, expected_providers=1
         )
 
-        ignore_paths: List[str] = [
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['created-at'\]",
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['snapshot-id'\]",
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['manifest-list'\]",
-        ]
-
         # Verify the output.
         mce_helpers.check_golden_file(
             pytestconfig,
-            ignore_paths=ignore_paths,
+            ignore_paths=PATHS_IN_GOLDEN_FILE_TO_IGNORE,
             output_path=deleted_mces_path,
             golden_path=test_resources_dir / "iceberg_deleted_table_mces_golden.json",
         )
@@ -202,16 +227,11 @@ def test_iceberg_profiling(docker_compose_runner, pytestconfig, tmp_path, mock_t
         run_datahub_cmd(
             ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
         )
-        # These paths change from one instance run of the clickhouse docker to the other, and the FROZEN_TIME does not apply to these.
-        ignore_paths: List[str] = [
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['created-at'\]",
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['snapshot-id'\]",
-            r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['manifest-list'\]",
-        ]
+
         # Verify the output.
         mce_helpers.check_golden_file(
             pytestconfig,
-            ignore_paths=ignore_paths,
+            ignore_paths=PATHS_IN_GOLDEN_FILE_TO_IGNORE,
             output_path=tmp_path / "iceberg_mces.json",
             golden_path=test_resources_dir / "iceberg_profile_mces_golden.json",
         )
diff --git a/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml b/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml
index 380df845e737c..cde21d85ed2d9 100644
--- a/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml
+++ b/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml
@@ -3,6 +3,7 @@ run_id: kafka-test
 source:
   type: kafka
   config:
+    ingest_schemas_as_entities: true
     connection:
       bootstrap: "localhost:29092"
       schema_registry_url: "http://localhost:28081"
diff --git a/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml b/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml
new file mode 100644
index 0000000000000..34cf8fd47e658
--- /dev/null
+++ b/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml
@@ -0,0 +1,20 @@
+run_id: kafka-test
+
+source:
+  type: kafka
+  config:
+    connection:
+      bootstrap: "localhost:29092"
+      schema_registry_url: "http://localhost:28081"
+      consumer_config:
+        security.protocol: "SASL_PLAINTEXT"
+        sasl.mechanism: "OAUTHBEARER"
+        oauth_cb: "oauth:create_token"
+    domain:
+      "urn:li:domain:sales":
+        allow:
+          - "key_value_topic"
+sink:
+  type: file
+  config:
+    filename: "./kafka_mces.json"
diff --git a/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json
new file mode 100644
index 0000000000000..7810c8077b31d
--- /dev/null
+++ b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json
@@ -0,0 +1,575 @@
+[
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "key_topic",
+                        "platform": "urn:li:dataPlatform:kafka",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "44fd7a7b325d6fdd4275b1f02a79c1a8",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.KafkaSchema": {
+                                "documentSchema": "",
+                                "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}",
+                                "keySchemaType": "AVRO"
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "id",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "namespace",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/kafka"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Partitions": "1",
+                            "Replication Factor": "1",
+                            "min.insync.replicas": "1",
+                            "retention.bytes": "-1",
+                            "retention.ms": "604800000",
+                            "cleanup.policy": "delete",
+                            "max.message.bytes": "1048588",
+                            "unclean.leader.election.enable": "false"
+                        },
+                        "name": "key_topic",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Topic"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "key_value_topic",
+                        "platform": "urn:li:dataPlatform:kafka",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "a79a2fe3adab60b21d272a9cc3e93595",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.KafkaSchema": {
+                                "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}",
+                                "documentSchemaType": "AVRO",
+                                "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}",
+                                "keySchemaType": "AVRO"
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "id",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "namespace",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "email",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Email"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"tags\": [\"Email\"]}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "firstName",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Name"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"tags\": [\"Name\"]}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "lastName",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Name"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"tags\": [\"Name\"]}"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/kafka"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:PII"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Partitions": "1",
+                            "Replication Factor": "1",
+                            "min.insync.replicas": "1",
+                            "retention.bytes": "-1",
+                            "retention.ms": "604800000",
+                            "cleanup.policy": "delete",
+                            "max.message.bytes": "1048588",
+                            "unclean.leader.election.enable": "false",
+                            "Schema Name": "key_value_topic-value"
+                        },
+                        "name": "key_value_topic",
+                        "description": "Value schema for kafka topic",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Topic"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "domains",
+    "aspect": {
+        "json": {
+            "domains": [
+                "urn:li:domain:sales"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "value_topic",
+                        "platform": "urn:li:dataPlatform:kafka",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "62c7c400ec5760797a59c45e59c2f2dc",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.KafkaSchema": {
+                                "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}",
+                                "documentSchemaType": "AVRO",
+                                "keySchema": "\"string\"",
+                                "keySchemaType": "AVRO"
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "[version=2.0].[key=True].[type=string]",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "email",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Email"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"tags\": [\"Email\"]}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "firstName",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Name"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"tags\": [\"Name\"]}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "lastName",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Name"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"tags\": [\"Name\"]}"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/kafka"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:PII"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Partitions": "1",
+                            "Replication Factor": "1",
+                            "min.insync.replicas": "1",
+                            "retention.bytes": "-1",
+                            "retention.ms": "604800000",
+                            "cleanup.policy": "delete",
+                            "max.message.bytes": "1048588",
+                            "unclean.leader.election.enable": "false",
+                            "Schema Name": "value_topic-value"
+                        },
+                        "name": "value_topic",
+                        "description": "Value schema for kafka topic",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Topic"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:Email",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "Email"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:Name",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "Name"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:PII",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "PII"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "kafka-test",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml
new file mode 100644
index 0000000000000..7f44e43c3c490
--- /dev/null
+++ b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml
@@ -0,0 +1,16 @@
+run_id: kafka-test
+
+source:
+  type: kafka
+  config:
+    connection:
+      bootstrap: "localhost:29092"
+      schema_registry_url: "http://localhost:28081"
+    domain:
+      "urn:li:domain:sales":
+        allow:
+          - "key_value_topic"
+sink:
+  type: file
+  config:
+    filename: "./kafka_without_schemas_mces.json"
diff --git a/metadata-ingestion/tests/integration/kafka/oauth.py b/metadata-ingestion/tests/integration/kafka/oauth.py
new file mode 100644
index 0000000000000..81a91fcd5e406
--- /dev/null
+++ b/metadata-ingestion/tests/integration/kafka/oauth.py
@@ -0,0 +1,30 @@
+import logging
+from typing import Any, Tuple
+
+logger = logging.getLogger(__name__)
+
+MESSAGE: str = "OAuth token `create_token` callback"
+
+
+def create_token(*args: Any, **kwargs: Any) -> Tuple[str, int]:
+    logger.warning(MESSAGE)
+    return (
+        "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRfaWQiOiJrYWZrYV9jbGllbnQiLCJleHAiOjE2OTg3NjYwMDB9.dummy_sig_abcdef123456",
+        3600,
+    )
+
+
+def create_token_no_args() -> Tuple[str, int]:
+    logger.warning(MESSAGE)
+    return (
+        "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRfaWQiOiJrYWZrYV9jbGllbnQiLCJleHAiOjE2OTg3NjYwMDB9.dummy_sig_abcdef123456",
+        3600,
+    )
+
+
+def create_token_only_kwargs(**kwargs: Any) -> Tuple[str, int]:
+    logger.warning(MESSAGE)
+    return (
+        "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRfaWQiOiJrYWZrYV9jbGllbnQiLCJleHAiOjE2OTg3NjYwMDB9.dummy_sig_abcdef123456",
+        3600,
+    )
diff --git a/metadata-ingestion/tests/integration/kafka/test_kafka.py b/metadata-ingestion/tests/integration/kafka/test_kafka.py
index dfdbea5de5cbf..648c4b26b20a7 100644
--- a/metadata-ingestion/tests/integration/kafka/test_kafka.py
+++ b/metadata-ingestion/tests/integration/kafka/test_kafka.py
@@ -1,10 +1,15 @@
+import logging
 import subprocess
 
 import pytest
+import yaml
 from freezegun import freeze_time
 
+from datahub.configuration.common import ConfigurationError
 from datahub.ingestion.api.source import SourceCapability
-from datahub.ingestion.source.kafka import KafkaSource
+from datahub.ingestion.run.pipeline import Pipeline
+from datahub.ingestion.source.kafka.kafka import KafkaSource, KafkaSourceConfig
+from tests.integration.kafka import oauth  # type: ignore
 from tests.test_helpers import mce_helpers, test_connection_helpers
 from tests.test_helpers.click_helpers import run_datahub_cmd
 from tests.test_helpers.docker_helpers import wait_for_port
@@ -38,20 +43,21 @@ def mock_kafka_service(docker_compose_runner, test_resources_dir):
         yield docker_compose_runner
 
 
+@pytest.mark.parametrize("approach", ["kafka_without_schemas", "kafka"])
 @freeze_time(FROZEN_TIME)
 @pytest.mark.integration
 def test_kafka_ingest(
-    mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time
+    mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time, approach
 ):
     # Run the metadata ingestion pipeline.
-    config_file = (test_resources_dir / "kafka_to_file.yml").resolve()
+    config_file = (test_resources_dir / f"{approach}_to_file.yml").resolve()
     run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path)
 
     # Verify the output.
     mce_helpers.check_golden_file(
         pytestconfig,
-        output_path=tmp_path / "kafka_mces.json",
-        golden_path=test_resources_dir / "kafka_mces_golden.json",
+        output_path=tmp_path / f"{approach}_mces.json",
+        golden_path=test_resources_dir / f"{approach}_mces_golden.json",
         ignore_paths=[],
     )
 
@@ -96,6 +102,88 @@ def test_kafka_test_connection(mock_kafka_service, config_dict, is_success):
         test_connection_helpers.assert_capability_report(
             capability_report=report.capability_report,
             failure_capabilities={
-                SourceCapability.SCHEMA_METADATA: "Failed to establish a new connection"
+                SourceCapability.SCHEMA_METADATA: "[Errno 111] Connection refused"
             },
         )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_kafka_oauth_callback(
+    mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time
+):
+    # Run the metadata ingestion pipeline.
+    config_file = (test_resources_dir / "kafka_to_file_oauth.yml").resolve()
+
+    log_file = tmp_path / "kafka_oauth_message.log"
+
+    file_handler = logging.FileHandler(
+        str(log_file)
+    )  # Add a file handler to later validate a test-case
+    logging.getLogger().addHandler(file_handler)
+
+    recipe: dict = {}
+    with open(config_file) as fp:
+        recipe = yaml.safe_load(fp)
+
+    pipeline = Pipeline.create(recipe)
+
+    pipeline.run()
+
+    # Initialize flags to track oauth events
+    checks = {
+        "consumer_polling": False,
+        "consumer_oauth_callback": False,
+        "admin_polling": False,
+        "admin_oauth_callback": False,
+    }
+
+    # Read log file and check for oauth events
+    with open(log_file, "r") as file:
+        for line in file:
+            # Check for polling events
+            if "Initiating polling for kafka admin client" in line:
+                checks["admin_polling"] = True
+            elif "Initiating polling for kafka consumer" in line:
+                checks["consumer_polling"] = True
+
+            # Check for oauth callbacks
+            if oauth.MESSAGE in line:
+                if checks["consumer_polling"] and not checks["admin_polling"]:
+                    checks["consumer_oauth_callback"] = True
+                elif checks["consumer_polling"] and checks["admin_polling"]:
+                    checks["admin_oauth_callback"] = True
+
+    # Verify all oauth events occurred
+    assert checks["consumer_polling"], "Consumer polling was not initiated"
+    assert checks["consumer_oauth_callback"], "Consumer oauth callback not found"
+    assert checks["admin_polling"], "Admin polling was not initiated"
+    assert checks["admin_oauth_callback"], "Admin oauth callback not found"
+
+
+def test_kafka_source_oauth_cb_signature():
+    with pytest.raises(
+        ConfigurationError,
+        match=("oauth_cb function must accept single positional argument."),
+    ):
+        KafkaSourceConfig.parse_obj(
+            {
+                "connection": {
+                    "bootstrap": "foobar:9092",
+                    "consumer_config": {"oauth_cb": "oauth:create_token_no_args"},
+                }
+            }
+        )
+
+    with pytest.raises(
+        ConfigurationError,
+        match=("oauth_cb function must accept single positional argument."),
+    ):
+        KafkaSourceConfig.parse_obj(
+            {
+                "connection": {
+                    "bootstrap": "foobar:9092",
+                    "consumer_config": {"oauth_cb": "oauth:create_token_only_kwargs"},
+                }
+            }
+        )
diff --git a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json
index 5cac7b1bb73b1..6ae772c134cb3 100644
--- a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json
+++ b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json
@@ -11,6 +11,7 @@
                         "description": "lorem ipsum",
                         "charts": [],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -440,7 +441,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "bogus data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/bogus data/my_view",
@@ -616,7 +620,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -835,6 +842,62 @@
         "pipelineName": "stateful-looker-pipeline"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{\"1\": \"test-1@looker.com\", \"2\": \"test-2@looker.com\", \"3\": \"test-3@looker.com\"}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "stateful-looker-pipeline"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "stateful-looker-pipeline"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "stateful-looker-pipeline"
+    }
+},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
index 24a738a815cda..d7620980a9ced 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
@@ -11,6 +11,7 @@
                         "description": "lorem ipsum",
                         "charts": [],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -282,7 +283,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -493,6 +497,59 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
index b1460779da4f5..13963af55bfe5 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
@@ -202,6 +202,7 @@
                             "urn:li:chart:(looker,dashboard_elements.2)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -520,7 +521,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "looker_hub",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -731,6 +735,59 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:Dimension",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
index 74400b9b5cc56..f11d060102851 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
@@ -202,6 +202,7 @@
                             "urn:li:chart:(looker,dashboard_elements.2)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -520,7 +521,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "looker_hub",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -731,6 +735,59 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{\"1\": \"test-1@looker.com\", \"2\": \"test-2@looker.com\", \"3\": \"test-3@looker.com\"}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:Dimension",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_folder_path_pattern_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_folder_path_pattern_ingest.json
index 89241fb52fb63..f6e39dd5286cd 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_folder_path_pattern_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_folder_path_pattern_ingest.json
@@ -287,6 +287,7 @@
                         "description": "third",
                         "charts": [],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -613,7 +614,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -824,6 +828,59 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{\"1\": \"test-1@looker.com\", \"2\": \"test-2@looker.com\", \"3\": \"test-3@looker.com\"}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
index f178e97e78fa0..203bed843155c 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
@@ -210,6 +210,7 @@
                             "urn:li:chart:(looker,dashboard_elements.2)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -463,6 +464,21 @@
                             "/Folders/Shared"
                         ]
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:test-1@looker.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
                 }
             ]
         }
@@ -707,6 +723,21 @@
                             "/Folders/Personal"
                         ]
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:test-2@looker.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
                 }
             ]
         }
@@ -1124,7 +1155,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -1300,7 +1334,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "order_model",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "order_explore",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/order_model/order_explore",
@@ -1476,7 +1513,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "sales_model",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "sales_explore",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/sales_model/sales_explore",
@@ -1695,6 +1735,62 @@
         "pipelineName": "execution-1"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{\"1\": \"test-1@looker.com\", \"2\": \"test-2@looker.com\", \"3\": \"test-3@looker.com\"}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "execution-1"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "execution-1"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "execution-1"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:Dimension",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json
index d969ef62a96e5..87af50f95ed6b 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json
@@ -229,6 +229,7 @@
                             "urn:li:chart:(looker,ap-south-1.dashboard_elements.2)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -574,7 +575,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -789,6 +793,60 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:8436a2a37c4a7e81fb08c9c8415d2e4b",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:8436a2a37c4a7e81fb08c9c8415d2e4b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:looker,ap-south-1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:8436a2a37c4a7e81fb08c9c8415d2e4b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:Dimension",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
index 153db363c7828..b990ce7c67dab 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
@@ -202,6 +202,7 @@
                             "urn:li:chart:(looker,dashboard_elements.2)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -520,7 +521,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -755,6 +759,59 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:Dimension",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
index 98adbdc5b829e..391192b3d16f3 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
@@ -11,6 +11,7 @@
                         "description": "lorem ipsum",
                         "charts": [],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -282,7 +283,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -509,6 +513,59 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_non_personal_independent_look.json b/metadata-ingestion/tests/integration/looker/golden_test_non_personal_independent_look.json
index 63ffdda8c5b6f..4909a6af73a22 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_non_personal_independent_look.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_non_personal_independent_look.json
@@ -210,6 +210,7 @@
                             "urn:li:chart:(looker,dashboard_elements.2)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -463,6 +464,21 @@
                             "/Folders/Shared"
                         ]
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:test-1@looker.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
                 }
             ]
         }
@@ -783,7 +799,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -959,7 +978,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "sales_model",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "sales_explore",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/sales_model/sales_explore",
@@ -1178,6 +1200,62 @@
         "pipelineName": "execution-1"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{\"1\": \"test-1@looker.com\", \"2\": \"test-2@looker.com\", \"3\": \"test-3@looker.com\"}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "execution-1"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "execution-1"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "execution-1"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:Dimension",
diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
index 567ab78a14754..ddeb5428b1d72 100644
--- a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
+++ b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
@@ -210,6 +210,7 @@
                             "urn:li:chart:(looker,dashboard_elements.2)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -539,7 +540,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -758,6 +762,62 @@
         "pipelineName": "stateful-looker-pipeline"
     }
 },
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{\"1\": \"test-1@looker.com\", \"2\": \"test-2@looker.com\", \"3\": \"test-3@looker.com\"}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "stateful-looker-pipeline"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "stateful-looker-pipeline"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "stateful-looker-pipeline"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:Dimension",
@@ -810,8 +870,8 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -827,8 +887,8 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -844,8 +904,8 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -861,8 +921,8 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
index 3befb62a631de..ed0c5401c9029 100644
--- a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
+++ b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
@@ -1,4 +1,66 @@
 [
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(looker,dashboard_elements.3)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {
+                            "upstream_fields": ""
+                        },
+                        "title": "",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 0,
+                                "actor": "urn:li:corpuser:unknown"
+                            },
+                            "lastModified": {
+                                "time": 0,
+                                "actor": "urn:li:corpuser:unknown"
+                            }
+                        },
+                        "chartUrl": "https://looker.company.com/x/",
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(looker,dashboard_elements.3)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Look"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
@@ -9,8 +71,11 @@
                         "customProperties": {},
                         "title": "foo",
                         "description": "lorem ipsum",
-                        "charts": [],
+                        "charts": [
+                            "urn:li:chart:(looker,dashboard_elements.3)"
+                        ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,
@@ -88,6 +153,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(looker,dashboard_elements.3)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(looker,dashboards.1)",
@@ -214,6 +295,98 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "looker",
+                "env": "PROD",
+                "model_name": "look_data"
+            },
+            "name": "look_data",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "LookML Model"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Explore"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@@ -234,7 +407,10 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "data",
                             "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "my_view",
                             "looker.explore.file": "test_source_file.lkml"
                         },
                         "externalUrl": "https://looker.company.com/explore/data/my_view",
@@ -385,6 +561,180 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/Explore/look_data"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "project": "lkml_samples",
+                            "model": "look_data",
+                            "looker.explore.label": "My Explore View",
+                            "looker.explore.name": "look_view",
+                            "looker.explore.file": "test_source_file.lkml"
+                        },
+                        "externalUrl": "https://looker.company.com/explore/look_data/look_view",
+                        "name": "My Explore View",
+                        "description": "lorem ipsum",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
+                        "upstreams": [
+                            {
+                                "auditStamp": {
+                                    "time": 1586847600000,
+                                    "actor": "urn:li:corpuser:datahub"
+                                },
+                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.underlying_view,PROD)",
+                                "type": "VIEW"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "look_view",
+                        "platform": "urn:li:dataPlatform:looker",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "dim1",
+                                "nullable": false,
+                                "description": "dimension one description",
+                                "label": "Dimensions One Label",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ],
+                        "primaryKeys": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Explore"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "embed",
+    "aspect": {
+        "json": {
+            "renderUrl": "https://looker.company.com/embed/explore/look_data/look_view"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Explore"
+                },
+                {
+                    "id": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb",
+                    "urn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
@@ -675,8 +1025,45 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)",
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "platformResourceInfo",
+    "aspect": {
+        "json": {
+            "resourceType": "USER_ID_MAPPING",
+            "primaryKey": "",
+            "value": {
+                "blob": "{\"1\": \"test-1@looker.com\", \"2\": \"test-2@looker.com\", \"3\": \"test-3@looker.com\"}",
+                "contentType": "JSON"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:1cec84235c544a141e63dd2077da2562",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -692,7 +1079,7 @@
 },
 {
     "entityType": "chart",
-    "entityUrn": "urn:li:chart:(looker,dashboard_elements.3)",
+    "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py
index 7238a49cb37d2..c96bcc729a95d 100644
--- a/metadata-ingestion/tests/integration/looker/test_looker.py
+++ b/metadata-ingestion/tests/integration/looker/test_looker.py
@@ -31,7 +31,10 @@
 from datahub.ingestion.api.source import SourceReport
 from datahub.ingestion.run.pipeline import Pipeline, PipelineInitError
 from datahub.ingestion.source.looker import looker_common, looker_usage
-from datahub.ingestion.source.looker.looker_common import LookerExplore
+from datahub.ingestion.source.looker.looker_common import (
+    LookerDashboardSourceReport,
+    LookerExplore,
+)
 from datahub.ingestion.source.looker.looker_config import LookerCommonConfig
 from datahub.ingestion.source.looker.looker_lib_wrapper import (
     LookerAPI,
@@ -83,6 +86,7 @@ def test_looker_ingest(pytestconfig, tmp_path, mock_time):
     with mock.patch("looker_sdk.init40") as mock_sdk:
         mock_sdk.return_value = mocked_client
         setup_mock_dashboard(mocked_client)
+        mocked_client.run_inline_query.side_effect = side_effect_query_inline
         setup_mock_explore(mocked_client)
 
         test_resources_dir = pytestconfig.rootpath / "tests/integration/looker"
@@ -319,6 +323,7 @@ def setup_mock_look(mocked_client):
     mocked_client.all_looks.return_value = [
         Look(
             id="1",
+            user_id="1",
             title="Outer Look",
             description="I am not part of any Dashboard",
             query_id="1",
@@ -327,6 +332,7 @@ def setup_mock_look(mocked_client):
         Look(
             id="2",
             title="Personal Look",
+            user_id="2",
             description="I am not part of any Dashboard and in personal folder",
             query_id="2",
             folder=FolderBase(
@@ -411,7 +417,9 @@ def setup_mock_dashboard_multiple_charts(mocked_client):
     )
 
 
-def setup_mock_dashboard_with_usage(mocked_client):
+def setup_mock_dashboard_with_usage(
+    mocked_client: mock.MagicMock, skip_look: bool = False
+) -> None:
     mocked_client.all_dashboards.return_value = [Dashboard(id="1")]
     mocked_client.dashboard.return_value = Dashboard(
         id="1",
@@ -434,7 +442,13 @@ def setup_mock_dashboard_with_usage(mocked_client):
                 ),
             ),
             DashboardElement(
-                id="3", type="", look=LookWithQuery(id="3", view_count=30)
+                id="3",
+                type="" if skip_look else "vis",  # Looks only ingested if type == `vis`
+                look=LookWithQuery(
+                    id="3",
+                    view_count=30,
+                    query=Query(model="look_data", view="look_view"),
+                ),
             ),
         ],
     )
@@ -561,6 +575,20 @@ def get_user(
     mocked_client.user.side_effect = get_user
 
 
+def setup_mock_all_user(mocked_client):
+    def all_users(
+        fields: Optional[str] = None,
+        transport_options: Optional[transport.TransportOptions] = None,
+    ) -> List[User]:
+        return [
+            User(id="1", email="test-1@looker.com"),
+            User(id="2", email="test-2@looker.com"),
+            User(id="3", email="test-3@looker.com"),
+        ]
+
+    mocked_client.all_users.side_effect = all_users
+
+
 def side_effect_query_inline(
     result_format: str, body: WriteQuery, transport_options: Optional[TransportOptions]
 ) -> str:
@@ -594,6 +622,12 @@ def side_effect_query_inline(
                     HistoryViewField.HISTORY_DASHBOARD_USER: 1,
                     HistoryViewField.HISTORY_DASHBOARD_RUN_COUNT: 5,
                 },
+                {
+                    HistoryViewField.HISTORY_DASHBOARD_ID: "5",
+                    HistoryViewField.HISTORY_CREATED_DATE: "2022-07-07",
+                    HistoryViewField.HISTORY_DASHBOARD_USER: 1,
+                    HistoryViewField.HISTORY_DASHBOARD_RUN_COUNT: 5,
+                },
             ]
         ),
         looker_usage.QueryId.DASHBOARD_PER_USER_PER_DAY_USAGE_STAT: json.dumps(
@@ -714,6 +748,7 @@ def test_looker_ingest_usage_history(pytestconfig, tmp_path, mock_time):
         mocked_client.run_inline_query.side_effect = side_effect_query_inline
         setup_mock_explore(mocked_client)
         setup_mock_user(mocked_client)
+        setup_mock_all_user(mocked_client)
 
         test_resources_dir = pytestconfig.rootpath / "tests/integration/looker"
 
@@ -772,6 +807,70 @@ def test_looker_ingest_usage_history(pytestconfig, tmp_path, mock_time):
         )
 
 
+@freeze_time(FROZEN_TIME)
+def test_looker_filter_usage_history(pytestconfig, tmp_path, mock_time):
+    mocked_client = mock.MagicMock()
+    with mock.patch("looker_sdk.init40") as mock_sdk:
+        mock_sdk.return_value = mocked_client
+        setup_mock_dashboard_with_usage(mocked_client, skip_look=True)
+        mocked_client.run_inline_query.side_effect = side_effect_query_inline
+        setup_mock_explore(mocked_client)
+        setup_mock_user(mocked_client)
+
+        temp_output_file = f"{tmp_path}/looker_mces.json"
+        pipeline = Pipeline.create(
+            {
+                "run_id": "looker-test",
+                "source": {
+                    "type": "looker",
+                    "config": {
+                        "base_url": "https://looker.company.com",
+                        "client_id": "foo",
+                        "client_secret": "bar",
+                        "extract_usage_history": True,
+                        "max_threads": 1,
+                    },
+                },
+                "sink": {
+                    "type": "file",
+                    "config": {
+                        "filename": temp_output_file,
+                    },
+                },
+            }
+        )
+        pipeline.run()
+        pipeline.pretty_print_summary()
+        pipeline.raise_from_status()
+
+        # There should be 4 dashboardUsageStatistics aspects (one absolute and 3 timeseries)
+        dashboard_usage_aspect_count = 0
+        # There should be 0 chartUsageStatistics -- filtered by set of ingested charts
+        chart_usage_aspect_count = 0
+        with open(temp_output_file) as f:
+            temp_output_dict = json.load(f)
+            for element in temp_output_dict:
+                if (
+                    element.get("entityType") == "dashboard"
+                    and element.get("aspectName") == "dashboardUsageStatistics"
+                ):
+                    dashboard_usage_aspect_count = dashboard_usage_aspect_count + 1
+                if (
+                    element.get("entityType") == "chart"
+                    and element.get("aspectName") == "chartUsageStatistics"
+                ):
+                    chart_usage_aspect_count = chart_usage_aspect_count + 1
+
+        assert dashboard_usage_aspect_count == 4
+        assert chart_usage_aspect_count == 0
+
+        source_report = cast(LookerDashboardSourceReport, pipeline.source.get_report())
+        # From timeseries query
+        assert str(source_report.dashboards_skipped_for_usage) == str(["5"])
+        # From dashboard element
+        assert str(source_report.charts_skipped_for_usage) == str(["3"])
+
+
 @freeze_time(FROZEN_TIME)
 def test_looker_ingest_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
     output_file_name: str = "looker_mces.json"
@@ -946,6 +1045,8 @@ def ingest_independent_looks(
         mock_sdk.return_value = mocked_client
         setup_mock_dashboard(mocked_client)
         setup_mock_explore(mocked_client)
+        setup_mock_user(mocked_client)
+        setup_mock_all_user(mocked_client)
         setup_mock_look(mocked_client)
 
         test_resources_dir = pytestconfig.rootpath / "tests/integration/looker"
@@ -1047,7 +1148,6 @@ def test_independent_soft_deleted_looks(
     mocked_client = mock.MagicMock()
 
     with mock.patch("looker_sdk.init40") as mock_sdk:
-
         mock_sdk.return_value = mocked_client
         setup_mock_look(mocked_client)
         setup_mock_soft_deleted_look(mocked_client)
diff --git a/metadata-ingestion/tests/integration/lookml/gms_schema_resolution/data.model.lkml b/metadata-ingestion/tests/integration/lookml/gms_schema_resolution/data.model.lkml
new file mode 100644
index 0000000000000..95391f6a73e63
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/gms_schema_resolution/data.model.lkml
@@ -0,0 +1,6 @@
+connection: "my_connection"
+
+include: "top_10_employee_income_source.view.lkml"
+
+explore: top_10_employee_income_source {
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/gms_schema_resolution/top_10_employee_income_source.view.lkml b/metadata-ingestion/tests/integration/lookml/gms_schema_resolution/top_10_employee_income_source.view.lkml
new file mode 100644
index 0000000000000..6037bab33c44f
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/gms_schema_resolution/top_10_employee_income_source.view.lkml
@@ -0,0 +1,18 @@
+view: top_10_employee_income_source {
+   sql_table_name: "db.public.employee"
+    ;;
+  dimension: id {
+    type: number
+    sql: ${TABLE}.id ;;
+  }
+
+  dimension: name {
+    type: string
+    sql: ${TABLE}.name ;;
+  }
+
+  dimension: source {
+    type: string
+    sql: ${TABLE}.source ;;
+  }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/gms_schema_resolution_golden.json b/metadata-ingestion/tests/integration/lookml/gms_schema_resolution_golden.json
new file mode 100644
index 0000000000000..9b0dd78ca1e8e
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/gms_schema_resolution_golden.json
@@ -0,0 +1,358 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "looker",
+                "env": "PROD",
+                "project_name": "lkml_samples"
+            },
+            "name": "lkml_samples",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:looker"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "LookML Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Folders"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "view: top_10_employee_income_source {\n   sql_table_name: \"db.public.employee\"\n    ;;\n  dimension: id {\n    type: number\n    sql: ${TABLE}.id ;;\n  }\n\n  dimension: name {\n    type: string\n    sql: ${TABLE}.name ;;\n  }\n\n  dimension: source {\n    type: string\n    sql: ${TABLE}.source ;;\n  }\n}",
+            "viewLanguage": "lookml"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/Develop/lkml_samples/"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
+                        "upstreams": [
+                            {
+                                "auditStamp": {
+                                    "time": 1586847600000,
+                                    "actor": "urn:li:corpuser:datahub"
+                                },
+                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,public.employee,PROD)",
+                                "type": "VIEW"
+                            }
+                        ],
+                        "fineGrainedLineages": [
+                            {
+                                "upstreamType": "FIELD_SET",
+                                "upstreams": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,public.employee,PROD),Id)"
+                                ],
+                                "downstreamType": "FIELD",
+                                "downstreams": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD),id)"
+                                ],
+                                "confidenceScore": 1.0
+                            },
+                            {
+                                "upstreamType": "FIELD_SET",
+                                "upstreams": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,public.employee,PROD),Name)"
+                                ],
+                                "downstreamType": "FIELD",
+                                "downstreams": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD),name)"
+                                ],
+                                "confidenceScore": 1.0
+                            },
+                            {
+                                "upstreamType": "FIELD_SET",
+                                "upstreams": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,public.employee,PROD),source)"
+                                ],
+                                "downstreamType": "FIELD",
+                                "downstreams": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD),source)"
+                                ],
+                                "confidenceScore": 1.0
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "top_10_employee_income_source",
+                        "platform": "urn:li:dataPlatform:looker",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "description": "",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "number",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "description": "",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "source",
+                                "nullable": false,
+                                "description": "",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ],
+                        "primaryKeys": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "looker.file.path": "top_10_employee_income_source.view.lkml",
+                            "looker.model": "data"
+                        },
+                        "name": "top_10_employee_income_source",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Develop"
+                },
+                {
+                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
+                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:Dimension",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "Dimension"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json
deleted file mode 100644
index 5b39e8dd96ac2..0000000000000
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json
+++ /dev/null
@@ -1,2820 +0,0 @@
-[
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-    "changeType": "UPSERT",
-    "aspectName": "containerProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "platform": "looker",
-                "env": "PROD",
-                "project_name": "lkml_samples"
-            },
-            "name": "lkml_samples",
-            "env": "PROD"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-    "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
-    "aspect": {
-        "json": {
-            "platform": "urn:li:dataPlatform:looker"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "LookML Project"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Folders"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "SELECT\n          is_latest,\n          country,\n          city,\n          timestamp,\n          measurement\n        FROM\n          my_table",
-            "viewLanguage": "sql"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),country)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),city)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),is_latest)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),timestamp)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),measurement)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "my_view",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "country",
-                                "nullable": false,
-                                "description": "The country",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "city",
-                                "nullable": false,
-                                "description": "City",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "is_latest",
-                                "nullable": false,
-                                "description": "Is latest data",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
-                                    }
-                                },
-                                "nativeDataType": "yesno",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "timestamp",
-                                "nullable": false,
-                                "description": "Timestamp of measurement",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
-                                    }
-                                },
-                                "nativeDataType": "time",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        },
-                                        {
-                                            "tag": "urn:li:tag:Temporal"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "average_measurement",
-                                "nullable": false,
-                                "description": "My measurement",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "average",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Measure"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "foo.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "my_view",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "SELECT\n          country,\n          city,\n          timestamp,\n          measurement\n        FROM\n          ${my_view.SQL_TABLE_NAME} AS my_view",
-            "viewLanguage": "sql"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "my_derived_view",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "country",
-                                "nullable": false,
-                                "description": "The country",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "city",
-                                "nullable": false,
-                                "description": "City",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "timestamp",
-                                "nullable": false,
-                                "description": "Timestamp of measurement",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
-                                    }
-                                },
-                                "nativeDataType": "time",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        },
-                                        {
-                                            "tag": "urn:li:tag:Temporal"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "average_measurement",
-                                "nullable": false,
-                                "description": "My measurement",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "average",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Measure"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "bar.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "my_derived_view",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "view: include_able_view {\n  sql_table_name: looker_schema.include_able ;;\n}\n",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.include_able,PROD)",
-                                "type": "VIEW"
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "included_view_file.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "include_able_view",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "include: \"/included_view_file.view\"\n\nview: looker_events {\n  sql_table_name: looker_schema.events ;;\n}\n\nview: extending_looker_events {\n  extends: [looker_events]\n\n  measure: additional_measure {\n    type: count\n  }\n}\n\nview: autodetect_sql_name_based_on_view_name {}\n\nview: test_include_external_view {\n  extends: [include_able_view]\n}\n",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.events,PROD)",
-                                "type": "VIEW"
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "view_declarations.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "looker_events",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "include: \"/included_view_file.view\"\n\nview: looker_events {\n  sql_table_name: looker_schema.events ;;\n}\n\nview: extending_looker_events {\n  extends: [looker_events]\n\n  measure: additional_measure {\n    type: count\n  }\n}\n\nview: autodetect_sql_name_based_on_view_name {}\n\nview: test_include_external_view {\n  extends: [include_able_view]\n}\n",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.events,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.events,PROD),additional_measure)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD),additional_measure)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "extending_looker_events",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "additional_measure",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "count",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Measure"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "view_declarations.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "extending_looker_events",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "include: \"/included_view_file.view\"\n\nview: looker_events {\n  sql_table_name: looker_schema.events ;;\n}\n\nview: extending_looker_events {\n  extends: [looker_events]\n\n  measure: additional_measure {\n    type: count\n  }\n}\n\nview: autodetect_sql_name_based_on_view_name {}\n\nview: test_include_external_view {\n  extends: [include_able_view]\n}\n",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.autodetect_sql_name_based_on_view_name,PROD)",
-                                "type": "VIEW"
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "view_declarations.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "autodetect_sql_name_based_on_view_name",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "include: \"/included_view_file.view\"\n\nview: looker_events {\n  sql_table_name: looker_schema.events ;;\n}\n\nview: extending_looker_events {\n  extends: [looker_events]\n\n  measure: additional_measure {\n    type: count\n  }\n}\n\nview: autodetect_sql_name_based_on_view_name {}\n\nview: test_include_external_view {\n  extends: [include_able_view]\n}\n",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.include_able,PROD)",
-                                "type": "VIEW"
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "view_declarations.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "test_include_external_view",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "SELECT date AS DATE,\n         platform  AS aliased_platform,\n         country",
-            "viewLanguage": "sql"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/nested"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),date)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),platform)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),country)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "fragment_derived_view",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "date",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NullType": {}
-                                    }
-                                },
-                                "nativeDataType": "unknown",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "aliased_platform",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NullType": {}
-                                    }
-                                },
-                                "nativeDataType": "unknown",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "country",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NullType": {}
-                                    }
-                                },
-                                "nativeDataType": "unknown",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "nested/fragment_derived.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "fragment_derived_view",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                },
-                {
-                    "id": "nested"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "SELECT\n            customer_id,\n            SUM(sale_price) AS lifetime_spend\n          FROM\n            order\n          WHERE\n            {% condition order_region %} order.region {% endcondition %}\n          GROUP BY 1",
-            "viewLanguage": "sql"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),customer_id)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),sale_price)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "customer_facts",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "customer_id",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NullType": {}
-                                    }
-                                },
-                                "nativeDataType": "unknown",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "lifetime_spend",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NullType": {}
-                                    }
-                                },
-                                "nativeDataType": "unknown",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "liquid.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "customer_facts",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "view: ability {\n  sql_table_name: \"ECOMMERCE\".\"ABILITY\"\n    ;;\n\n  dimension: pk {\n    type: number\n    sql: ${TABLE}.\"PK\" ;;\n  }\n\n  measure: count {\n    type: count\n    drill_fields: []\n  }\n}\n",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.ecommerce.ability,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.ecommerce.ability,PROD),pk)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD),pk)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.ecommerce.ability,PROD),count)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD),count)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "ability",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "pk",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "number",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "count",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "count",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Measure"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "ability.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "ability",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "view: owners {\n  dimension: id {\n    primary_key: yes\n    sql: ${TABLE}.id ;;\n  }\n  dimension: owner_name {\n    sql: ${TABLE}.owner_name ;;\n  }\n}",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.owners,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.owners,PROD),id)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD),id)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.owners,PROD),owner_name)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD),owner_name)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "owners",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "id",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": true
-                            },
-                            {
-                                "fieldPath": "owner_name",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": [
-                            "id"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "owners.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "owners",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "explore_source: my_view_explore {\n  bind_all_filters: yes\n\n  column: country {\n    field: my_view_explore.country\n  }\n\n  column: city {\n    field: my_view_explore.city\n  }\n\n  column: is_latest {\n    field: my_view_explore.is_latest\n  }\n\n  derived_column: derived_col {\n    sql: coalesce(country, 'US') ;;\n  }\n}",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD),my_view_explore.country)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD),country)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD),my_view_explore.city)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD),city)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD),my_view_explore.country)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD),unique_countries)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD),my_view_explore.is_latest)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD),derived_col)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "view_derived_explore",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "country",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "city",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "unique_countries",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "count_distinct",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Measure"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "derived_col",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "sum",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Measure"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "native_derived_table.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "view_derived_explore",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "view: flights {\n  sql_table_name: flightstats.accidents ;;\n\n  dimension: id {\n    label: \"id\"\n    primary_key: yes\n    type: number\n    sql: ${TABLE}.id ;;\n  }\n}\n\n# override type of id parameter\nview: +flights {\n  dimension: id {\n    label: \"id\"\n    primary_key: yes\n    type: string\n    sql: ${TABLE}.id ;;\n  }\n}",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/Develop/lkml_samples/"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.flightstats.accidents,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.flightstats.accidents,PROD),id)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD),id)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "flights",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "id",
-                                "nullable": false,
-                                "description": "",
-                                "label": "id",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "number",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": true
-                            }
-                        ],
-                        "primaryKeys": [
-                            "id"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "flights.view.lkml",
-                            "looker.model": "data"
-                        },
-                        "name": "flights",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "Develop"
-                },
-                {
-                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
-                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
-            "urn": "urn:li:tag:Dimension",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.tag.TagProperties": {
-                        "name": "Dimension",
-                        "description": "A tag that is applied to all dimension fields."
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
-            "urn": "urn:li:tag:Temporal",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.tag.TagProperties": {
-                        "name": "Temporal",
-                        "description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations."
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
-            "urn": "urn:li:tag:Measure",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.tag.TagProperties": {
-                        "name": "Measure",
-                        "description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on"
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "tag",
-    "entityUrn": "urn:li:tag:Dimension",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "tag",
-    "entityUrn": "urn:li:tag:Measure",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "tag",
-    "entityUrn": "urn:li:tag:Temporal",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test",
-        "lastRunId": "no-run-id-provided"
-    }
-}
-]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py
index e4eb564e3e86b..940e7f36675f7 100644
--- a/metadata-ingestion/tests/integration/lookml/test_lookml.py
+++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py
@@ -1,8 +1,8 @@
 import logging
 import pathlib
-from typing import Any, List
+from typing import Any, List, Optional, Tuple
 from unittest import mock
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import pydantic
 import pytest
@@ -10,24 +10,22 @@
 from freezegun import freeze_time
 from looker_sdk.sdk.api40.models import DBConnection
 
-from datahub.configuration.common import PipelineExecutionError
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.ingestion.source.file import read_metadata_file
+from datahub.ingestion.source.looker.looker_dataclasses import LookerModel
 from datahub.ingestion.source.looker.looker_template_language import (
     SpecialVariable,
     load_and_preprocess_file,
     resolve_liquid_variable,
 )
-from datahub.ingestion.source.looker.lookml_source import (
-    LookerModel,
-    LookerRefinementResolver,
-    LookMLSourceConfig,
-)
+from datahub.ingestion.source.looker.lookml_config import LookMLSourceConfig
+from datahub.ingestion.source.looker.lookml_refinement import LookerRefinementResolver
 from datahub.metadata.schema_classes import (
     DatasetSnapshotClass,
     MetadataChangeEventClass,
     UpstreamLineageClass,
 )
+from datahub.sql_parsing.schema_resolver import SchemaInfo, SchemaResolver
 from tests.test_helpers import mce_helpers
 from tests.test_helpers.state_helpers import get_current_checkpoint_from_pipeline
 
@@ -65,7 +63,7 @@ def get_default_recipe(output_file_path, base_folder_path):
 
 @freeze_time(FROZEN_TIME)
 def test_lookml_ingest(pytestconfig, tmp_path, mock_time):
-    """Test backwards compatibility with previous form of config with new flags turned off"""
+    """Test backwards compatibility with a previous form of config with new flags turned off"""
     test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml"
     mce_out_file = "expected_output.json"
 
@@ -518,53 +516,6 @@ def ingestion_test(
         )
 
 
-@freeze_time(FROZEN_TIME)
-def test_lookml_bad_sql_parser(pytestconfig, tmp_path, mock_time):
-    """Incorrect specification of sql parser should not fail ingestion"""
-    test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml"
-    mce_out = "lookml_mces_badsql_parser.json"
-    pipeline = Pipeline.create(
-        {
-            "run_id": "lookml-test",
-            "source": {
-                "type": "lookml",
-                "config": {
-                    "base_folder": str(test_resources_dir / "lkml_samples"),
-                    "connection_to_platform_map": {
-                        "my_connection": {
-                            "platform": "snowflake",
-                            "default_db": "default_db",
-                            "default_schema": "default_schema",
-                        }
-                    },
-                    "parse_table_names_from_sql": True,
-                    "project_name": "lkml_samples",
-                    "sql_parser": "bad.sql.Parser",
-                    "emit_reachable_views_only": False,
-                    "process_refinements": False,
-                },
-            },
-            "sink": {
-                "type": "file",
-                "config": {
-                    "filename": f"{tmp_path}/{mce_out}",
-                },
-            },
-        }
-    )
-    pipeline.run()
-    pipeline.pretty_print_summary()
-    pipeline.raise_from_status(raise_warnings=False)
-    with pytest.raises(PipelineExecutionError):  # we expect the source to have warnings
-        pipeline.raise_from_status(raise_warnings=True)
-
-    mce_helpers.check_golden_file(
-        pytestconfig,
-        output_path=tmp_path / mce_out,
-        golden_path=test_resources_dir / mce_out,
-    )
-
-
 @freeze_time(FROZEN_TIME)
 def test_lookml_git_info(pytestconfig, tmp_path, mock_time):
     """Add github info to config"""
@@ -881,7 +832,6 @@ def test_manifest_parser(pytestconfig: pytest.Config) -> None:
 
 @freeze_time(FROZEN_TIME)
 def test_duplicate_field_ingest(pytestconfig, tmp_path, mock_time):
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml"
     mce_out_file = "duplicate_ingest_mces_output.json"
 
@@ -940,7 +890,7 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t
 
 @freeze_time(FROZEN_TIME)
 def test_special_liquid_variables():
-    text: str = """
+    text: str = """{% assign source_table_variable = "source_table" | sql_quote | non_existing_filter_where_it_should_not_fail %}
         SELECT
           employee_id,
           employee_name,
@@ -954,7 +904,7 @@ def test_special_liquid_variables():
             'default_table' as source
           {% endif %},
           employee_income
-        FROM source_table
+        FROM {{ source_table_variable }}
     """
     input_liquid_variable: dict = {}
 
@@ -1009,7 +959,7 @@ def test_special_liquid_variables():
     expected_text: str = (
         "\n        SELECT\n          employee_id,\n          employee_name,\n          \n            "
         "prod_core.data.r_metric_summary_v2\n          ,\n          employee_income\n        FROM "
-        "source_table\n    "
+        "'source_table'\n    "
     )
     assert actual_text == expected_text
 
@@ -1064,3 +1014,40 @@ def test_drop_hive(pytestconfig, tmp_path, mock_time):
         output_path=tmp_path / mce_out_file,
         golden_path=golden_path,
     )
+
+
+@freeze_time(FROZEN_TIME)
+def test_gms_schema_resolution(pytestconfig, tmp_path, mock_time):
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml"
+    mce_out_file = "drop_hive_dot.json"
+
+    new_recipe = get_default_recipe(
+        f"{tmp_path}/{mce_out_file}",
+        f"{test_resources_dir}/gms_schema_resolution",
+    )
+
+    new_recipe["source"]["config"]["connection_to_platform_map"] = {
+        "my_connection": "hive"
+    }
+
+    return_value: Tuple[str, Optional[SchemaInfo]] = (
+        "fake_dataset_urn",
+        {
+            "Id": "String",
+            "Name": "String",
+            "source": "String",
+        },
+    )
+
+    with patch.object(SchemaResolver, "resolve_urn", return_value=return_value):
+        pipeline = Pipeline.create(new_recipe)
+        pipeline.run()
+        pipeline.pretty_print_summary()
+        pipeline.raise_from_status(raise_warnings=True)
+
+    golden_path = test_resources_dir / "gms_schema_resolution_golden.json"
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=tmp_path / mce_out_file,
+        golden_path=golden_path,
+    )
diff --git a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json
index ed00dc5734680..84dbdbe89f7b5 100644
--- a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json
+++ b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json
@@ -176,6 +176,7 @@
                         "datasets": [
                             "urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)"
                         ],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 1639169724316,
@@ -253,6 +254,8 @@
     "aspect": {
         "json": {
             "customProperties": {
+                "id": "19780522",
+                "created_at": "2024-09-02T07:38:43.755Z",
                 "updated_at": "2024-09-02T07:40:44.046Z",
                 "last_run_id": "3535709679",
                 "data_source_id": "44763",
@@ -260,7 +263,6 @@
             },
             "externalUrl": "https://app.mode.com/acryl/datasets/24f66e1701b6",
             "name": "Dataset 1",
-            "description": "### Source Code\n``` sql\n-- Returns first 100 rows from DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY\n  SELECT \n\t\tAGE,\n\t\tID,\n\t\tNAME,\n\t\t_FIVETRAN_DELETED,\n\t\t_FIVETRAN_SYNCED\n FROM DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY LIMIT 100;\n\n-- Returns first 100 rows from ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER\n  SELECT \n\t\tCOMMUNICATION_ACCOUNT_ID,\n\t\tID,\n\t\tMMS_CAPABLE,\n\t\tPHONE_NUMBER,\n\t\tSMS_CAPABLE,\n\t\tSTATUS,\n\t\tSTATUS_TLM,\n\t\tTLM,\n\t\tVOICE_CAPABLE,\n\t\tWHEN_CREATED\n FROM ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER LIMIT 100;\n \n \n```\n            ",
             "tags": []
         }
     },
@@ -270,6 +272,24 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "-- Returns first 100 rows from DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY\n  SELECT \n\t\tAGE,\n\t\tID,\n\t\tNAME,\n\t\t_FIVETRAN_DELETED,\n\t\t_FIVETRAN_SYNCED\n FROM DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY LIMIT 100;\n\n-- Returns first 100 rows from ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER\n  SELECT \n\t\tCOMMUNICATION_ACCOUNT_ID,\n\t\tID,\n\t\tMMS_CAPABLE,\n\t\tPHONE_NUMBER,\n\t\tSMS_CAPABLE,\n\t\tSTATUS,\n\t\tSTATUS_TLM,\n\t\tTLM,\n\t\tVOICE_CAPABLE,\n\t\tWHEN_CREATED\n FROM ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER LIMIT 100;\n \n ",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "mode-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)",
@@ -336,13 +356,14 @@
     "aspect": {
         "json": {
             "customProperties": {
+                "id": "10149707",
+                "created_at": "2021-12-10T20:55:24.361Z",
                 "updated_at": "2021-12-10T23:12:53.273Z",
                 "last_run_id": "1897576958",
                 "data_source_id": "34499"
             },
             "externalUrl": "https://app.mode.com/acryl/reports/9d2da37fa91e/details/queries/6e26a9f3d4e2",
             "name": "Customer and staff",
-            "description": "### Source Code\n``` sql\nSELECT  rental.*,  staff.first_name \"Staff First Name\", staff.last_name \"Staff Last Name\" FROM  {{  @join_on_definition    as                      rental  }} join staff on staff.staff_id = rental.staff_id where selected_id = {{ selected_id }} \n{% form %}\nselected_id:\n   type: text\n   default: my_id\n{% endform %}\n```\n            ",
             "tags": []
         }
     },
@@ -352,6 +373,24 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,10149707,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT  rental.*,  staff.first_name \"Staff First Name\", staff.last_name \"Staff Last Name\" FROM  {{  @join_on_definition    as                      rental  }} join staff on staff.staff_id = rental.staff_id where selected_id = {{ selected_id }} \n{% form %}\nselected_id:\n   type: text\n   default: my_id\n{% endform %}",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "mode-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,10149707,PROD)",
diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
index 2666990ed1333..103977463a8f2 100644
--- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
+++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
@@ -93,6 +93,7 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
+            "name": "emptyCollection",
             "customProperties": {},
             "tags": []
         }
@@ -503,6 +504,7 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
+            "name": "firstCollection",
             "customProperties": {},
             "tags": []
         }
@@ -4201,6 +4203,7 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
+            "name": "largeCollection",
             "customProperties": {
                 "schema.downsampled": "True",
                 "schema.totalFields": "501"
@@ -4444,6 +4447,7 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
+            "name": "secondCollection",
             "customProperties": {},
             "tags": []
         }
diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json
index 577a053f1c741..5cf1ff47a3866 100644
--- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json
+++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json
@@ -93,6 +93,7 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
+            "name": "emptyCollection",
             "customProperties": {},
             "tags": []
         }
@@ -359,6 +360,7 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
+            "name": "firstCollection",
             "customProperties": {
                 "schema.downsampled": "True",
                 "schema.totalFields": "22"
@@ -580,6 +582,7 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
+            "name": "largeCollection",
             "customProperties": {
                 "schema.downsampled": "True",
                 "schema.totalFields": "502"
@@ -823,6 +826,7 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
+            "name": "secondCollection",
             "customProperties": {},
             "tags": []
         }
diff --git a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json
index f820efc739949..e026664a78e0b 100644
--- a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json
+++ b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json
@@ -60,15 +60,7 @@
                 "op": "add",
                 "path": "/inputDatajobEdges/urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)",
                 "value": {
-                    "destinationUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)",
-                    "created": {
-                        "time": 1638532800000,
-                        "actor": "urn:li:corpuser:datahub"
-                    },
-                    "lastModified": {
-                        "time": 1638532800000,
-                        "actor": "urn:li:corpuser:datahub"
-                    }
+                    "destinationUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)"
                 }
             }
         ]
@@ -178,30 +170,14 @@
                 "op": "add",
                 "path": "/inputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)",
                 "value": {
-                    "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)",
-                    "created": {
-                        "time": 1638532800000,
-                        "actor": "urn:li:corpuser:datahub"
-                    },
-                    "lastModified": {
-                        "time": 1638532800000,
-                        "actor": "urn:li:corpuser:datahub"
-                    }
+                    "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)"
                 }
             },
             {
                 "op": "add",
                 "path": "/inputDatajobEdges/urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)",
                 "value": {
-                    "destinationUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)",
-                    "created": {
-                        "time": 1638532800000,
-                        "actor": "urn:li:corpuser:datahub"
-                    },
-                    "lastModified": {
-                        "time": 1638532800000,
-                        "actor": "urn:li:corpuser:datahub"
-                    }
+                    "destinationUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)"
                 }
             }
         ]
@@ -287,15 +263,7 @@
                 "op": "add",
                 "path": "/inputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)",
                 "value": {
-                    "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)",
-                    "created": {
-                        "time": 1638532800000,
-                        "actor": "urn:li:corpuser:datahub"
-                    },
-                    "lastModified": {
-                        "time": 1638532800000,
-                        "actor": "urn:li:corpuser:datahub"
-                    }
+                    "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)"
                 }
             }
         ]
diff --git a/metadata-ingestion/tests/integration/okta/test_okta.py b/metadata-ingestion/tests/integration/okta/test_okta.py
index 63ef8793caddd..10148273c9366 100644
--- a/metadata-ingestion/tests/integration/okta/test_okta.py
+++ b/metadata-ingestion/tests/integration/okta/test_okta.py
@@ -58,14 +58,12 @@ def run_ingest(
     mocked_functions_reference,
     recipe,
 ):
-
     with patch(
         "datahub.ingestion.source.identity.okta.OktaClient"
     ) as MockClient, patch(
         "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph",
         mock_datahub_graph,
     ) as mock_checkpoint:
-
         mock_checkpoint.return_value = mock_datahub_graph
 
         mocked_functions_reference(MockClient=MockClient)
@@ -277,7 +275,6 @@ def overwrite_group_in_mocked_data(test_resources_dir, MockClient):
 def _init_mock_okta_client(
     test_resources_dir, MockClient, mock_users_json=None, mock_groups_json=None
 ):
-
     okta_users_json_file = (
         test_resources_dir / "okta_users.json"
         if mock_users_json is None
diff --git a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json
index ad270857dd7fc..ffa8d43a6a5ab 100755
--- a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json
+++ b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json
@@ -20,7 +20,7 @@
                     "com.linkedin.pegasus2avro.common.InstitutionalMemory": {
                         "elements": [
                             {
-                                "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/",
+                                "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/",
                                 "description": "Link to call for the dataset.",
                                 "createStamp": {
                                     "time": 1586847600,
@@ -71,7 +71,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "openapi-2020_04_14-07_00_00"
+        "runId": "openapi-2020_04_14-07_00_00-m1k7d5",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -95,7 +96,7 @@
                     "com.linkedin.pegasus2avro.common.InstitutionalMemory": {
                         "elements": [
                             {
-                                "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/v2",
+                                "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/v2",
                                 "description": "Link to call for the dataset.",
                                 "createStamp": {
                                     "time": 1586847600,
@@ -146,7 +147,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "openapi-2020_04_14-07_00_00"
+        "runId": "openapi-2020_04_14-07_00_00-m1k7d5",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +163,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "openapi-2020_04_14-07_00_00"
+        "runId": "openapi-2020_04_14-07_00_00-m1k7d5",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -176,7 +179,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "openapi-2020_04_14-07_00_00"
+        "runId": "openapi-2020_04_14-07_00_00-m1k7d5",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml b/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml
index e9dc03482c2bc..8575c23239461 100644
--- a/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml
+++ b/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml
@@ -2,8 +2,8 @@ source:
   type: openapi
   config:
     name: test_openapi
-    url: https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/
-    swagger_file: v3.0/api-with-examples.yaml
+    url: https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/
+    swagger_file: v3.0/pass/api-with-examples.yaml
 
 
 sink:
diff --git a/metadata-ingestion/tests/integration/oracle/common.py b/metadata-ingestion/tests/integration/oracle/common.py
index 79dbda8c30f89..9e2cc42ef1025 100644
--- a/metadata-ingestion/tests/integration/oracle/common.py
+++ b/metadata-ingestion/tests/integration/oracle/common.py
@@ -33,7 +33,6 @@ def scalar(self):
 
 @dataclass
 class MockConstraints:
-
     constraint_name: str = "mock constraint name"
     constraint_type: str = "P"
     local_column: str = "mock column name"
diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json
index cbcadde6feb21..abd9b2350638a 100644
--- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json
+++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json
@@ -17,7 +17,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -33,7 +33,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -49,7 +49,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -67,7 +67,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -83,7 +83,23 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:937a38ee28b69ecae38665c5e842d0ad",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0e497517e191d344b0c403231bc708d0"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -106,7 +122,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -122,7 +138,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -138,7 +154,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -156,23 +172,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:937a38ee28b69ecae38665c5e842d0ad",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:0e497517e191d344b0c403231bc708d0"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -193,7 +193,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -209,7 +209,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -272,7 +272,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -290,7 +290,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -315,7 +315,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -331,7 +331,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -394,7 +394,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -412,7 +412,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -437,7 +437,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -453,7 +453,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -519,7 +519,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -537,7 +537,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -555,7 +555,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -580,7 +580,23 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:1965527855ae77f259a8ddea2b8eed2f",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0e497517e191d344b0c403231bc708d0"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -603,7 +619,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -619,7 +635,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -635,7 +651,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -653,23 +669,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:1965527855ae77f259a8ddea2b8eed2f",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:0e497517e191d344b0c403231bc708d0"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -690,7 +690,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -706,7 +706,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -769,7 +769,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -787,7 +787,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -812,7 +812,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -828,7 +828,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -891,7 +891,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -909,7 +909,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -934,7 +934,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -950,7 +950,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -1016,7 +1016,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -1034,7 +1034,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -1052,7 +1052,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -1077,13 +1077,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_view,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -1106,7 +1106,7 @@
                     ],
                     "downstreamType": "FIELD",
                     "downstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_view,PROD),MOCK_COLUMN1)"
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN1)"
                     ],
                     "confidenceScore": 1.0
                 },
@@ -1117,7 +1117,7 @@
                     ],
                     "downstreamType": "FIELD",
                     "downstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_view,PROD),MOCK_COLUMN2)"
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN2)"
                     ],
                     "confidenceScore": 1.0
                 }
@@ -1126,13 +1126,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_view,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -1155,7 +1155,7 @@
                     ],
                     "downstreamType": "FIELD",
                     "downstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_view,PROD),MOCK_COLUMN1)"
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN1)"
                     ],
                     "confidenceScore": 1.0
                 },
@@ -1166,7 +1166,7 @@
                     ],
                     "downstreamType": "FIELD",
                     "downstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_view,PROD),MOCK_COLUMN2)"
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN2)"
                     ],
                     "confidenceScore": 1.0
                 }
@@ -1175,39 +1175,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "oracle-2022_02_03-07_00_00",
+        "runId": "oracle-2022_02_03-07_00_00-uzcdxn",
         "lastRunId": "no-run-id-provided"
     }
 }
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json
index 049008bddc58e..1394a140b0a24 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json
@@ -1,7 +1,7 @@
 [
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -17,7 +17,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -402,11 +402,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -445,7 +445,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -461,7 +461,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json
index fa4bcb8abaa94..b83127e6f13b0 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json
@@ -63,8 +63,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -138,8 +137,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -238,8 +236,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -338,8 +335,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -446,8 +442,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -546,8 +541,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -646,8 +640,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -746,8 +739,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -846,8 +838,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -884,7 +875,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -900,7 +891,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -1118,11 +1109,12 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -1135,12 +1127,11 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -1311,11 +1302,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -1416,8 +1407,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1491,8 +1481,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1591,8 +1580,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1691,8 +1679,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1799,8 +1786,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1899,8 +1885,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1937,7 +1922,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -1953,7 +1938,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -1994,14 +1979,19 @@
         "json": [
             {
                 "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715"
+                "path": "/title",
+                "value": "SalesMarketing"
             },
             {
                 "op": "add",
                 "path": "/description",
                 "value": "Acryl sales marketing report"
             },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715"
+            },
             {
                 "op": "add",
                 "path": "/lastModified",
@@ -2015,11 +2005,6 @@
                         "actor": "urn:li:corpuser:unknown"
                     }
                 }
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "SalesMarketing"
             }
         ]
     },
@@ -2089,11 +2074,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -2132,7 +2117,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2148,7 +2133,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_app_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_app_ingest.json
new file mode 100644
index 0000000000000..5988b14977552
--- /dev/null
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_app_ingest.json
@@ -0,0 +1,242 @@
+[
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,apps.2A4D0E82-E7A4-45B1-BD72-2A2CF82C9CB6)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "title": "Finance",
+            "description": "The finance app",
+            "charts": [],
+            "datasets": [],
+            "dashboards": [
+                {
+                    "destinationUrn": "urn:li:dashboard:(powerbi,dashboards.744B07E3-FAA7-4BD7-BD17-3220BF0F6301)"
+                },
+                {
+                    "destinationUrn": "urn:li:dashboard:(powerbi,reports.455AB99B-E110-46E6-90D3-F015CABD1156)"
+                }
+            ],
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 1547372813,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,apps.2A4D0E82-E7A4-45B1-BD72-2A2CF82C9CB6)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/Workspace For App Testing"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,apps.2A4D0E82-E7A4-45B1-BD72-2A2CF82C9CB6)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,apps.2A4D0E82-E7A4-45B1-BD72-2A2CF82C9CB6)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "App"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,apps.2A4D0E82-E7A4-45B1-BD72-2A2CF82C9CB6)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Workspace For App Testing"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A700E2C1-D008-42DF-AFCA-A70A87D0B2A3)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/Workspace For App Testing"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A700E2C1-D008-42DF-AFCA-A70A87D0B2A3)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "0"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "Workspace For App Testing"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "8F756DE6-26AD-45FF-A201-44276FF1F561"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A700E2C1-D008-42DF-AFCA-A70A87D0B2A3)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A700E2C1-D008-42DF-AFCA-A70A87D0B2A3)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/A700E2C1-D008-42DF-AFCA-A70A87D0B2A3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A700E2C1-D008-42DF-AFCA-A70A87D0B2A3)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Workspace For App Testing"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json
index 60b36897ed2e4..5881ad5de0bcb 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json
@@ -17,6 +17,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
@@ -43,10 +59,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -56,15 +74,51 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
+            },
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                }
             ]
         }
     },
@@ -74,6 +128,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
@@ -115,13 +185,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -134,13 +206,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -150,22 +219,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,PBI_TEST.TEST.TESTTABLE,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -214,6 +275,24 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
@@ -230,6 +309,24 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
@@ -238,8 +335,52 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,PBI_TEST.TEST.TESTTABLE,PROD)",
+                    "type": "TRANSFORMED"
+                }
             ]
         }
     },
@@ -300,6 +441,46 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "changeType": "UPSERT",
+    "aspectName": "chartInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+            },
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
@@ -341,8 +522,8 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -360,13 +541,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -376,15 +554,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -395,18 +572,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
-            "description": "Library dataset description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -416,21 +589,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-test-project.universal.D_WH_DATE,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "PowerBI Tile"
             ]
         }
     },
@@ -441,13 +607,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "removed": false
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -458,14 +626,20 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-test-project.universal.D_WH_DATE,PROD)",
+                    "type": "TRANSFORMED"
+                }
             ]
         }
     },
@@ -476,15 +650,17 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
-            "viewLanguage": "m_query"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -497,51 +673,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD)",
-                    "type": "TRANSFORMED"
-                },
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_forecast,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ],
-            "fineGrainedLineages": [
-                {
-                    "upstreamType": "FIELD_SET",
-                    "upstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD),name)"
-                    ],
-                    "downstreamType": "FIELD",
-                    "downstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV),name)"
-                    ],
-                    "confidenceScore": 1.0
-                },
-                {
-                    "upstreamType": "FIELD_SET",
-                    "upstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD),name)"
-                    ],
-                    "downstreamType": "FIELD",
-                    "downstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV),name)"
-                    ],
-                    "confidenceScore": 1.0
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -552,7 +689,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -561,7 +698,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "job-history",
+            "name": "snowflake native-query-with-join",
             "description": "Library dataset description",
             "tags": []
         }
@@ -573,13 +710,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "removed": false
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -590,15 +729,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -608,22 +744,84 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
     "aspect": {
-        "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "demo-workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,salesdb.HR.EMPLOYEES,PROD)",
-                    "type": "TRANSFORMED"
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
                 }
-            ]
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -634,13 +832,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
             "viewLanguage": "m_query"
         }
     },
@@ -652,17 +850,17 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "ms_sql_native_table",
+            "description": "hr pbi test description",
             "tags": []
         }
     },
@@ -674,12 +872,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -690,15 +894,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -709,20 +910,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -734,14 +928,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
-            "viewLanguage": "m_query"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -754,16 +948,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -773,13 +961,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "removed": false
+            "username": "User1@foo.com"
         }
     },
     "systemMetadata": {
@@ -789,16 +977,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "username": "User2@foo.com"
         }
     },
     "systemMetadata": {
@@ -808,15 +993,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -829,16 +1012,45 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "hr pbi test description",
-            "tags": []
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),client_director)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),month_wid)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV),cd_agent_key)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),client_manager_closing_month)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),month_wid)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV),agent_key)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -851,18 +1063,11 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,library.dbo.book_issue,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -873,13 +1078,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "removed": false
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -890,14 +1096,52 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_forecast,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV),name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV),name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
             ]
         }
     },
@@ -909,7 +1153,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -920,7 +1164,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,library.dbo.book_issue,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -933,13 +1177,27 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
+    "aspectName": "ownership",
     "aspect": {
         "json": {
-            "username": "User1@foo.com"
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:User1@foo.com",
+                    "type": "NONE"
+                },
+                {
+                    "owner": "urn:li:corpuser:User2@foo.com",
+                    "type": "NONE"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
         }
     },
     "systemMetadata": {
@@ -949,13 +1207,17 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "username": "User2@foo.com"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -965,50 +1227,20 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
-                },
+            "upstreams": [
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,salesdb.HR.EMPLOYEES,PROD)",
+                    "type": "TRANSFORMED"
                 }
             ]
         }
@@ -1020,13 +1252,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1036,70 +1270,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-    "changeType": "UPSERT",
-    "aspectName": "chartKey",
-    "aspect": {
-        "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePaths",
-    "aspect": {
-        "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1109,70 +1288,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "chartKey",
-    "aspect": {
-        "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1182,14 +1310,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Tile"
+                "Table"
             ]
         }
     },
@@ -1200,15 +1328,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1218,17 +1344,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1238,14 +1366,14 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -1256,77 +1384,8 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
-    "changeType": "PATCH",
-    "aspectName": "dashboardInfo",
-    "aspect": {
-        "json": [
-            {
-                "op": "add",
-                "path": "/customProperties/chartCount",
-                "value": "2"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceName",
-                "value": "demo-workspace"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C10-4684-8180-826122881108"
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "test_dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/description",
-                "value": "Description of test dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-            },
-            {
-                "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://localhost/dashboards/web/1"
-            },
-            {
-                "op": "add",
-                "path": "/lastModified",
-                "value": {
-                    "created": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "lastModified": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    }
-                }
-            }
-        ]
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1341,14 +1400,19 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "customProperties": {
+                "datasetId": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff/details",
+            "name": "employee_ctc",
+            "description": "Employee Management",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1358,27 +1422,22 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "ownership",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
-                    "type": "NONE"
-                },
+            "upstreams": [
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
-                    "type": "NONE"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)",
+                    "type": "TRANSFORMED"
                 }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
+            ]
         }
     },
     "systemMetadata": {
@@ -1388,16 +1447,14 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -1425,22 +1482,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)",
@@ -1458,38 +1499,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff/details",
-            "name": "employee_ctc",
-            "description": "Employee Management",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1500,7 +1516,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
index b43e4a6c2c1c2..50058405a7aec 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
@@ -8,7 +8,10 @@
         "json": {
             "customProperties": {
                 "platform": "powerbi",
-                "workspace": "demo-workspace"
+                "workspace": "demo-workspace",
+                "workspace_id": "64ED5CAD-7C10-4684-8180-826122881108",
+                "workspace_name": "demo-workspace",
+                "workspace_type": "Workspace"
             },
             "name": "demo-workspace"
         }
@@ -122,15 +125,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "dummy",
-            "viewLanguage": "m_query"
+            "platform": "urn:li:dataPlatform:powerbi"
         }
     },
     "systemMetadata": {
@@ -140,19 +141,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "public issue_history",
-            "description": "Library dataset description",
-            "tags": []
+            "typeNames": [
+                "Semantic Model"
+            ]
         }
     },
     "systemMetadata": {
@@ -165,10 +162,10 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:powerbi"
+            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
         }
     },
     "systemMetadata": {
@@ -181,11 +178,14 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset"
+            "path": [
+                {
+                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
+                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                }
             ]
         }
     },
@@ -196,18 +196,15 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
-                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "dummy",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -217,13 +214,19 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "public issue_history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -256,8 +259,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -372,8 +374,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -488,8 +489,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -604,8 +604,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -720,8 +719,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -836,8 +834,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -952,8 +949,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1041,15 +1037,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
-            "viewLanguage": "m_query"
+            "platform": "urn:li:dataPlatform:powerbi"
         }
     },
     "systemMetadata": {
@@ -1059,19 +1053,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
-            "tags": []
+            "typeNames": [
+                "Semantic Model"
+            ]
         }
     },
     "systemMetadata": {
@@ -1084,10 +1074,10 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:powerbi"
+            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
         }
     },
     "systemMetadata": {
@@ -1100,11 +1090,14 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset"
+            "path": [
+                {
+                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
+                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                }
             ]
         }
     },
@@ -1115,18 +1108,15 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
-                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1136,13 +1126,19 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1175,8 +1171,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1291,8 +1286,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1345,7 +1339,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -1361,7 +1355,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -1811,11 +1805,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -1870,15 +1864,17 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "containerProperties",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "dummy",
-            "viewLanguage": "m_query"
+            "customProperties": {
+                "platform": "powerbi",
+                "dataset": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "name": "library-dataset"
         }
     },
     "systemMetadata": {
@@ -1888,19 +1884,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "public issue_history",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1910,13 +1900,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "removed": false
+            "platform": "urn:li:dataPlatform:powerbi"
         }
     },
     "systemMetadata": {
@@ -1926,15 +1916,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Semantic Model"
             ]
         }
     },
@@ -1944,15 +1933,31 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
+            "viewLogic": "dummy",
             "viewLanguage": "m_query"
         }
     },
@@ -1964,7 +1969,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -1973,7 +1978,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "SNOWFLAKE_TESTTABLE",
+            "name": "public issue_history",
             "description": "Library dataset description",
             "tags": []
         }
@@ -1986,7 +1991,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2002,14 +2007,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -2021,13 +2025,29 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
             "viewLanguage": "m_query"
         }
     },
@@ -2039,7 +2059,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -2048,7 +2068,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query",
+            "name": "SNOWFLAKE_TESTTABLE",
             "description": "Library dataset description",
             "tags": []
         }
@@ -2061,7 +2081,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2077,14 +2097,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -2096,13 +2115,29 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
             "viewLanguage": "m_query"
         }
     },
@@ -2114,7 +2149,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -2123,7 +2158,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
+            "name": "snowflake native-query",
             "description": "Library dataset description",
             "tags": []
         }
@@ -2136,7 +2171,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2152,14 +2187,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -2169,6 +2203,112 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "big-query-with-parameter",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
@@ -2233,8 +2373,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -2244,6 +2383,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
@@ -2308,8 +2463,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -2319,6 +2473,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
@@ -2383,8 +2553,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -2394,9 +2563,25 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -2412,7 +2597,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -2495,6 +2680,24 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PowerBI Page"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)",
@@ -2619,6 +2822,24 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PowerBI Page"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)",
@@ -2692,6 +2913,60 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "SalesMarketing"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Acryl sales marketing report"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)",
+                "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)",
+                "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)",
@@ -2752,11 +3027,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -2774,15 +3049,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Page"
-            ]
+            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
         }
     },
     "systemMetadata": {
@@ -2795,10 +3068,389 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "path": [
+                {
+                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
+                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "powerbi",
+                "dataset": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "name": "library-dataset"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:powerbi"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Semantic Model"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "dummy",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "public issue_history",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "SNOWFLAKE_TESTTABLE",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -2808,17 +3460,52 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
-                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-                }
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "big-query-with-parameter",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -2829,8 +3516,64 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2845,8 +3588,82 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2861,13 +3678,15 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User4@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "username": "User4@foo.com"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -2877,8 +3696,64 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User4@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2893,14 +3768,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Page"
+                "Table"
             ]
         }
     },
@@ -2911,13 +3786,13 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User3@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "username": "User3@foo.com"
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
         }
     },
     "systemMetadata": {
@@ -2927,13 +3802,15 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User3@foo.com",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "removed": false
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -2944,7 +3821,7 @@
 },
 {
     "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
     "changeType": "PATCH",
     "aspectName": "dashboardInfo",
     "aspect": {
@@ -2952,27 +3829,17 @@
             {
                 "op": "add",
                 "path": "/title",
-                "value": "SalesMarketing"
+                "value": "Printable SalesMarketing"
             },
             {
                 "op": "add",
                 "path": "/description",
                 "value": "Acryl sales marketing report"
             },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)",
-                "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)",
-                "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)"
-            },
             {
                 "op": "add",
                 "path": "/dashboardUrl",
-                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715"
+                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163"
             },
             {
                 "op": "add",
@@ -2997,8 +3864,8 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -3012,6 +3879,78 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/584cf13a-1485-41c2-a514-b1bb66fff163"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PaginatedReport"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
+                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
@@ -3021,7 +3960,10 @@
         "json": {
             "customProperties": {
                 "platform": "powerbi",
-                "workspace": "second-demo-workspace"
+                "workspace": "second-demo-workspace",
+                "workspace_id": "64ED5CAD-7C22-4684-8180-826122881108",
+                "workspace_name": "second-demo-workspace",
+                "workspace_type": "Workspace"
             },
             "name": "second-demo-workspace"
         }
@@ -3036,10 +3978,10 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -3083,15 +4025,13 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/second-demo-workspace"
-            ]
+            "path": []
         }
     },
     "systemMetadata": {
@@ -3101,13 +4041,29 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User3@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "removed": false
+            "username": "User3@foo.com"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User4@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "corpUserKey",
+    "aspect": {
+        "json": {
+            "username": "User4@foo.com"
         }
     },
     "systemMetadata": {
@@ -3120,14 +4076,11 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
-                    "urn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e"
-                }
+            "paths": [
+                "/powerbi/second-demo-workspace"
             ]
         }
     },
@@ -3195,11 +4148,10 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -3212,10 +4164,11 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "container": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e"
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -3233,11 +4186,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User3@foo.com",
+                    "owner": "urn:li:corpuser:User3@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User4@foo.com",
+                    "owner": "urn:li:corpuser:User4@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -3253,5 +4206,106 @@
         "runId": "powerbi-test",
         "lastRunId": "no-run-id-provided"
     }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
+                    "urn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User3@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User4@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cross_workspace_dataset.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cross_workspace_dataset.json
new file mode 100644
index 0000000000000..06286fcd5937a
--- /dev/null
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cross_workspace_dataset.json
@@ -0,0 +1,668 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "dummy",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)",
+    "changeType": "UPSERT",
+    "aspectName": "chartInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "317456E5-1FC7-4BDC-9C84-1185825E293D",
+                "datasetWebUrl": "http://localhost/groups/A8D655A6-F521-477E-8C22-255018583BF4/datasets/317456E5-1FC7-4BDC-9C84-1185825E293D/details"
+            },
+            "title": "Sale Order Tile",
+            "description": "Sale Order Tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "317456E5-1FC7-4BDC-9C84-1185825E293D"
+            },
+            "externalUrl": "http://localhost/groups/A8D655A6-F521-477E-8C22-255018583BF4/datasets/317456E5-1FC7-4BDC-9C84-1185825E293D/details",
+            "name": "public issue_history",
+            "description": "sales semantic model",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PowerBI Tile"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)",
+    "changeType": "UPSERT",
+    "aspectName": "chartKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "chartId": "charts.885D1762-1655-46BA-AFE3-74C6EC403A9E"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "dummy",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/Sales"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "FE362B98-956E-4394-BA37-6367EE6435E9"
+            },
+            "externalUrl": "http://localhost/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/datasets/FE362B98-956E-4394-BA37-6367EE6435E9/details",
+            "name": "core_sales_set",
+            "description": "base_records",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sales"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)",
+    "changeType": "UPSERT",
+    "aspectName": "chartInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "FE362B98-956E-4394-BA37-6367EE6435E9",
+                "datasetWebUrl": "http://localhost/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/datasets/FE362B98-956E-4394-BA37-6367EE6435E9/details"
+            },
+            "title": "Yearly Sales",
+            "description": "Yearly Sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PowerBI Tile"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)",
+    "changeType": "UPSERT",
+    "aspectName": "chartKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "chartId": "charts.945C2C2A-4588-45DE-8385-F24F5E39A57C"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/Sales"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sales"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)",
+    "changeType": "UPSERT",
+    "aspectName": "chartInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "0F0ADA0E-E38A-44F6-B667-90E93A96F5A1"
+            },
+            "title": "Not Present In Current Ingestion",
+            "description": "Not Present In Current Ingestion",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/Sales"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PowerBI Tile"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "3"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "Sales"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "A8D655A6-F521-477E-8C22-255018583BF4"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)",
+                "value": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)",
+                "value": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)",
+                "value": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)",
+    "changeType": "UPSERT",
+    "aspectName": "chartKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/Sales"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sales"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/A1C7204F-4D04-4E5E-B886-B30EA2C64CB3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Sales"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json
index c5414444cc35b..099f75a190ca2 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json
@@ -17,6 +17,40 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
@@ -40,8 +74,8 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -63,8 +97,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -75,15 +108,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
-            "viewLanguage": "m_query"
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -93,19 +126,17 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "SNOWFLAKE_TESTTABLE",
-            "description": "Library dataset description",
-            "tags": []
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -118,10 +149,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -131,16 +164,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -151,14 +182,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -169,7 +198,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -178,7 +207,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query",
+            "name": "SNOWFLAKE_TESTTABLE",
             "description": "Library dataset description",
             "tags": []
         }
@@ -193,10 +222,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -206,15 +237,36 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+            },
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                }
             ]
         }
     },
@@ -226,14 +278,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -244,7 +294,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -253,7 +303,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
+            "name": "snowflake native-query",
             "description": "Library dataset description",
             "tags": []
         }
@@ -266,12 +316,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -282,14 +334,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -300,15 +351,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -318,19 +369,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -341,12 +386,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -356,15 +403,51 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
+            },
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                }
             ]
         }
     },
@@ -376,14 +459,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -394,7 +475,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -403,7 +484,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "job-history",
+            "name": "big-query-with-parameter",
             "description": "Library dataset description",
             "tags": []
         }
@@ -415,13 +496,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -432,14 +515,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -450,15 +532,17 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
-            "viewLanguage": "m_query"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -468,19 +552,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
-            "tags": []
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -491,12 +570,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -506,15 +587,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -526,14 +606,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -544,17 +622,17 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
             "tags": []
         }
     },
@@ -566,12 +644,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -584,13 +664,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -603,12 +682,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -640,14 +717,67 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
     "aspect": {
-        "json": {
-            "removed": false
-        }
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "demo-workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -663,8 +793,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -675,52 +804,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -730,13 +820,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -746,14 +842,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Tile"
+                "Table"
             ]
         }
     },
@@ -764,14 +860,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -781,15 +878,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -799,17 +894,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -819,35 +911,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
-                },
+            "path": [
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                    "id": "demo-workspace"
                 }
             ]
         }
@@ -859,30 +931,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -892,14 +953,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Tile"
+                "Table"
             ]
         }
     },
@@ -910,15 +971,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -928,17 +989,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -948,15 +1007,13 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -966,77 +1023,8 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
-    "changeType": "PATCH",
-    "aspectName": "dashboardInfo",
-    "aspect": {
-        "json": [
-            {
-                "op": "add",
-                "path": "/customProperties/chartCount",
-                "value": "2"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceName",
-                "value": "demo-workspace"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C10-4684-8180-826122881108"
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "test_dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/description",
-                "value": "Description of test dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-            },
-            {
-                "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://localhost/dashboards/web/1"
-            },
-            {
-                "op": "add",
-                "path": "/lastModified",
-                "value": {
-                    "created": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "lastModified": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    }
-                }
-            }
-        ]
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1051,14 +1039,19 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1068,16 +1061,14 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json
index e1ddbfb901bad..07bb4f9aaad44 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json
@@ -40,13 +40,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:Promoted",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "tagKey",
     "aspect": {
         "json": {
-            "removed": false
+            "name": "Promoted"
         }
     },
     "systemMetadata": {
@@ -59,13 +59,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -78,13 +75,11 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "tags": [
-                {
-                    "tag": "urn:li:tag:Promoted"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -95,15 +90,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
-            "viewLanguage": "m_query"
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -114,18 +109,16 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "SNOWFLAKE_TESTTABLE",
-            "description": "Library dataset description",
-            "tags": []
+            "tags": [
+                {
+                    "tag": "urn:li:tag:Promoted"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -135,8 +128,8 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -154,50 +147,11 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "globalTags",
-    "aspect": {
-        "json": {
-            "tags": [
-                {
-                    "tag": "urn:li:tag:Promoted"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
-    "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
             "viewLanguage": "m_query"
         }
     },
@@ -209,7 +163,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -218,7 +172,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query",
+            "name": "SNOWFLAKE_TESTTABLE",
             "description": "Library dataset description",
             "tags": []
         }
@@ -230,13 +184,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -247,15 +203,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -265,15 +218,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "tags": [
+            "path": [
                 {
-                    "tag": "urn:li:tag:Promoted"
+                    "id": "demo-workspace"
                 }
             ]
         }
@@ -286,13 +239,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
             "viewLanguage": "m_query"
         }
     },
@@ -304,7 +257,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -313,7 +266,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
+            "name": "snowflake native-query",
             "description": "Library dataset description",
             "tags": []
         }
@@ -326,30 +279,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -361,16 +297,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "tags": [
-                {
-                    "tag": "urn:li:tag:Promoted"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -380,15 +312,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -399,18 +330,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
-            "description": "Library dataset description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -420,13 +347,37 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+            },
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -437,14 +388,15 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "tags": [
+                {
+                    "tag": "urn:li:tag:Promoted"
+                }
             ]
         }
     },
@@ -456,7 +408,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "globalTags",
     "aspect": {
@@ -476,13 +428,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
             "viewLanguage": "m_query"
         }
     },
@@ -494,7 +446,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -503,7 +455,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "job-history",
+            "name": "big-query-with-parameter",
             "description": "Library dataset description",
             "tags": []
         }
@@ -516,7 +468,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -531,15 +483,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -551,15 +502,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "tags": [
-                {
-                    "tag": "urn:li:tag:Promoted"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -570,37 +519,52 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
-            "viewLanguage": "m_query"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
-            "tags": []
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -610,8 +574,8 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -627,14 +591,15 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "tags": [
+                {
+                    "tag": "urn:li:tag:Promoted"
+                }
             ]
         }
     },
@@ -645,16 +610,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "tags": [
-                {
-                    "tag": "urn:li:tag:Promoted"
-                }
+            "typeNames": [
+                "PowerBI Tile"
             ]
         }
     },
@@ -666,13 +629,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
             "viewLanguage": "m_query"
         }
     },
@@ -684,17 +647,17 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
             "tags": []
         }
     },
@@ -705,13 +668,17 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -722,15 +689,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -758,19 +722,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "hr pbi test description",
-            "tags": []
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -799,13 +758,16 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "ms_sql_native_table",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -815,13 +777,15 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "username": "User1@foo.com"
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -831,13 +795,15 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "username": "User2@foo.com"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -847,51 +813,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -902,8 +831,8 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -918,15 +847,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "username": "User1@foo.com"
         }
     },
     "systemMetadata": {
@@ -936,14 +863,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "username": "User2@foo.com"
         }
     },
     "systemMetadata": {
@@ -953,15 +879,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -971,121 +901,10 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "chartInfo",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "chartKey",
-    "aspect": {
-        "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
             "paths": [
@@ -1100,34 +919,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -1206,6 +1005,26 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": [
+                {
+                    "tag": "urn:li:tag:Promoted"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
@@ -1239,6 +1058,26 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
@@ -1248,11 +1087,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -1270,15 +1109,15 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "path": [
+            "tags": [
                 {
-                    "id": "demo-workspace"
+                    "tag": "urn:li:tag:Promoted"
                 }
             ]
         }
@@ -1290,8 +1129,84 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1305,9 +1220,85 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": [
+                {
+                    "tag": "urn:li:tag:Promoted"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1322,13 +1313,13 @@
     }
 },
 {
-    "entityType": "tag",
-    "entityUrn": "urn:li:tag:Promoted",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "tagKey",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "name": "Promoted"
+            "removed": false
         }
     },
     "systemMetadata": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json b/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json
index d204d426a38d3..0b822ad19b425 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json
@@ -13,7 +13,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -34,7 +35,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -49,7 +51,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -60,14 +63,14 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json
index 6f899a7fa11b7..4bbd2e2b06542 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json
@@ -21,16 +21,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "public issue_history",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -43,10 +37,16 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "public issue_history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -63,8 +63,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -75,15 +74,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
-            "viewLanguage": "m_query"
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -93,19 +92,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "SNOWFLAKE_TESTTABLE",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -118,10 +111,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -131,15 +126,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "PowerBI Tile"
             ]
         }
     },
@@ -151,14 +145,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -169,7 +161,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -178,7 +170,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query",
+            "name": "SNOWFLAKE_TESTTABLE",
             "description": "Library dataset description",
             "tags": []
         }
@@ -193,10 +185,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -206,15 +200,16 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
             ]
         }
     },
@@ -226,14 +221,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -244,7 +237,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -253,7 +246,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
+            "name": "snowflake native-query",
             "description": "Library dataset description",
             "tags": []
         }
@@ -266,12 +259,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -282,14 +277,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -300,15 +294,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -318,19 +311,37 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
-            "description": "Library dataset description",
-            "tags": []
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -341,12 +352,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -357,15 +370,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -376,14 +386,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
-            "viewLanguage": "m_query"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "big-query-with-parameter",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -394,18 +408,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "job-history",
-            "description": "Library dataset description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -415,13 +425,52 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
+            },
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -431,15 +480,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -451,13 +499,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
             "viewLanguage": "m_query"
         }
     },
@@ -468,19 +516,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -491,7 +533,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -507,15 +549,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -526,13 +571,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
             "viewLanguage": "m_query"
         }
     },
@@ -546,16 +591,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
-            "tags": []
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -566,7 +607,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -582,15 +623,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "ms_sql_native_table",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -600,15 +644,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -621,16 +665,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "hr pbi test description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -641,7 +681,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -657,15 +697,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -676,7 +719,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -692,7 +735,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -707,51 +750,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -762,13 +768,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -781,12 +789,11 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -799,11 +806,14 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -813,8 +823,8 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
     "aspectName": "browsePaths",
     "aspect": {
@@ -831,18 +841,67 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
     "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "demo-workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
                 }
-            ]
-        }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -851,37 +910,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -891,13 +926,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "removed": false
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -907,14 +943,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -924,15 +961,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -942,15 +979,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -960,17 +995,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -983,11 +1020,13 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
             ]
         }
     },
@@ -998,67 +1037,16 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
-    "changeType": "PATCH",
-    "aspectName": "dashboardInfo",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
     "aspect": {
-        "json": [
-            {
-                "op": "add",
-                "path": "/customProperties/chartCount",
-                "value": "2"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceName",
-                "value": "demo-workspace"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C10-4684-8180-826122881108"
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "test_dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/description",
-                "value": "Description of test dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-            },
-            {
-                "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://localhost/dashboards/web/1"
-            },
-            {
-                "op": "add",
-                "path": "/lastModified",
-                "value": {
-                    "created": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "lastModified": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    }
-                }
-            }
-        ]
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -1067,8 +1055,8 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1083,14 +1071,19 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1108,11 +1101,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -1130,16 +1123,14 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -1151,7 +1142,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1167,7 +1158,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json
index efbd9abfdb911..a943472f307e5 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json
@@ -21,16 +21,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "public issue_history",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -43,10 +37,16 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "public issue_history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -63,8 +63,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -75,15 +74,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
-            "viewLanguage": "m_query"
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -93,19 +92,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "SNOWFLAKE_TESTTABLE",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -118,10 +111,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -131,15 +126,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "PowerBI Tile"
             ]
         }
     },
@@ -151,14 +145,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -169,7 +161,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -178,7 +170,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query",
+            "name": "SNOWFLAKE_TESTTABLE",
             "description": "Library dataset description",
             "tags": []
         }
@@ -193,10 +185,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -206,15 +200,16 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
             ]
         }
     },
@@ -226,14 +221,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -244,7 +237,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -253,7 +246,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
+            "name": "snowflake native-query",
             "description": "Library dataset description",
             "tags": []
         }
@@ -266,12 +259,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -282,14 +277,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -300,15 +294,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -318,19 +311,37 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
-            "description": "Library dataset description",
-            "tags": []
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -341,12 +352,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -357,15 +370,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -376,14 +386,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
-            "viewLanguage": "m_query"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "big-query-with-parameter",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -394,18 +408,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "job-history",
-            "description": "Library dataset description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -415,13 +425,52 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
+            },
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -431,15 +480,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -451,13 +499,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
             "viewLanguage": "m_query"
         }
     },
@@ -468,19 +516,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -491,7 +533,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -507,15 +549,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -526,13 +571,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
             "viewLanguage": "m_query"
         }
     },
@@ -546,16 +591,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
-            "tags": []
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -566,7 +607,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -582,15 +623,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "ms_sql_native_table",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -600,15 +644,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -621,16 +665,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "hr pbi test description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -641,7 +681,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -657,15 +697,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -676,7 +719,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -692,7 +735,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -707,51 +750,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -762,30 +768,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Tile"
+                "Table"
             ]
         }
     },
@@ -816,11 +806,13 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
             ]
         }
     },
@@ -831,16 +823,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -851,19 +841,25 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "dashboardInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+                "chartCount": "2",
+                "workspaceName": "demo-workspace",
+                "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
             },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
+            "title": "test_dashboard",
+            "description": "Description of test dashboard",
+            "charts": [
+                "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            ],
+            "datasets": [],
+            "dashboards": [],
             "lastModified": {
                 "created": {
                     "time": 0,
@@ -874,14 +870,7 @@
                     "actor": "urn:li:corpuser:unknown"
                 }
             },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
-                }
-            ]
+            "dashboardUrl": "https://localhost/dashboards/web/1"
         }
     },
     "systemMetadata": {
@@ -891,8 +880,8 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -907,15 +896,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -925,14 +913,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -942,15 +931,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -960,17 +949,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -980,15 +965,19 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1001,32 +990,14 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardInfo",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "chartCount": "2",
-                "workspaceName": "demo-workspace",
-                "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
-            },
-            "title": "test_dashboard",
-            "description": "Description of test dashboard",
-            "charts": [
-                "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-            ],
-            "datasets": [],
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
+            "path": [
+                {
+                    "id": "demo-workspace"
                 }
-            },
-            "dashboardUrl": "https://localhost/dashboards/web/1"
+            ]
         }
     },
     "systemMetadata": {
@@ -1036,8 +1007,26 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1052,14 +1041,19 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1077,11 +1071,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -1099,16 +1093,14 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -1120,7 +1112,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1136,7 +1128,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json
index 9a09cb4fec64d..df56bdc45db76 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json
@@ -17,6 +17,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-lineage-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
@@ -43,10 +59,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -56,15 +74,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -74,6 +91,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-lineage-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
@@ -115,13 +148,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -134,13 +169,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -150,20 +182,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "upstreams": [
+            "path": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.PBI_TEST.TEST.TESTTABLE,PROD)",
-                    "type": "TRANSFORMED"
+                    "id": "demo-workspace"
                 }
             ]
         }
@@ -216,12 +243,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -234,13 +263,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -250,22 +276,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -276,14 +294,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
-            "viewLanguage": "m_query"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -293,19 +311,37 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
-            "description": "Library dataset description",
-            "tags": []
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -316,12 +352,21 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "removed": false
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.PBI_TEST.TEST.TESTTABLE,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -332,14 +377,20 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4,PROD)",
+                    "type": "TRANSFORMED"
+                }
             ]
         }
     },
@@ -351,13 +402,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
             "viewLanguage": "m_query"
         }
     },
@@ -369,18 +420,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -391,12 +436,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "big-query-with-parameter",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -406,15 +457,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -426,28 +476,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.GSL_TEST_DB.PUBLIC.SALES_ANALYST,PROD)",
-                    "type": "TRANSFORMED"
-                },
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.GSL_TEST_DB.PUBLIC.SALES_FORECAST,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -458,15 +493,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -476,19 +509,52 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "job-history",
-            "description": "Library dataset description",
-            "tags": []
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
+            },
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -499,12 +565,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -515,15 +583,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -533,21 +604,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,high_performance_production_unit.salesdb.HR.EMPLOYEES,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "PowerBI Tile"
             ]
         }
     },
@@ -559,36 +623,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
-            "viewLanguage": "m_query"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-lineage-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -598,13 +638,17 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -615,14 +659,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -634,21 +677,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,operational_instance.mics.public.order_date,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -659,13 +695,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
             "viewLanguage": "m_query"
         }
     },
@@ -677,7 +713,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -686,7 +722,7 @@
                 "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
+            "name": "ms_sql_native_table",
             "description": "hr pbi test description",
             "tags": []
         }
@@ -701,29 +737,16 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-lineage-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -734,21 +757,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.library.dbo.book_issue,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -758,15 +772,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -779,16 +792,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "hr pbi test description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -799,7 +808,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -814,15 +823,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -859,7 +867,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -875,7 +883,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -890,50 +898,46 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
-                },
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-lineage-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.GSL_TEST_DB.PUBLIC.SALES_ANALYST,PROD)",
+                    "type": "TRANSFORMED"
                 },
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.GSL_TEST_DB.PUBLIC.SALES_FORECAST,PROD)",
+                    "type": "TRANSFORMED"
                 }
             ]
         }
@@ -945,14 +949,67 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
     "aspect": {
-        "json": {
-            "removed": false
-        }
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "demo-workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -961,14 +1018,21 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.library.dbo.book_issue,PROD)",
+                    "type": "TRANSFORMED"
+                }
             ]
         }
     },
@@ -979,14 +1043,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -996,15 +1059,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -1014,8 +1076,8 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -1034,37 +1096,27 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "ownership",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
+            "owners": [
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                    "owner": "urn:li:corpuser:User1@foo.com",
+                    "type": "NONE"
                 },
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                    "owner": "urn:li:corpuser:User2@foo.com",
+                    "type": "NONE"
                 }
-            ]
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
         }
     },
     "systemMetadata": {
@@ -1074,13 +1126,22 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "removed": false
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,high_performance_production_unit.salesdb.HR.EMPLOYEES,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1090,14 +1151,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1107,15 +1169,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1125,15 +1191,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1143,17 +1209,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1163,85 +1225,20 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-lineage-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
-    "changeType": "PATCH",
-    "aspectName": "dashboardInfo",
-    "aspect": {
-        "json": [
-            {
-                "op": "add",
-                "path": "/customProperties/chartCount",
-                "value": "2"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceName",
-                "value": "demo-workspace"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C10-4684-8180-826122881108"
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "test_dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/description",
-                "value": "Description of test dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-            },
-            {
-                "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://localhost/dashboards/web/1"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            {
-                "op": "add",
-                "path": "/lastModified",
-                "value": {
-                    "created": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "lastModified": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    }
-                }
-            }
-        ]
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
+        }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -1250,13 +1247,15 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -1266,14 +1265,15 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -1283,27 +1283,13 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "ownership",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
-                    "type": "NONE"
-                },
-                {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
-                    "type": "NONE"
-                }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1313,15 +1299,20 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
+            "upstreams": [
                 {
-                    "id": "demo-workspace"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,operational_instance.mics.public.order_date,PROD)",
+                    "type": "TRANSFORMED"
                 }
             ]
         }
@@ -1334,7 +1325,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1350,7 +1341,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json
index a4eb670a4b7f9..4c27dec3c6752 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json
@@ -18,13 +18,13 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "username": "User1@foo.com"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -55,22 +55,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
-    "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
-    "aspect": {
-        "json": {
-            "username": "User2@foo.com"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)",
@@ -91,7 +75,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -127,30 +111,16 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -161,14 +131,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -180,15 +149,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -198,13 +166,52 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
+            },
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -215,14 +222,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -235,12 +240,16 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
-            "viewLanguage": "m_query"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -253,16 +262,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
-            "description": "Library dataset description",
-            "tags": []
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -272,15 +277,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "PowerBI Tile"
             ]
         }
     },
@@ -308,7 +312,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -317,7 +321,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query",
+            "name": "big-query-with-parameter",
             "description": "Library dataset description",
             "tags": []
         }
@@ -330,15 +334,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -349,12 +352,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -365,7 +370,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -403,14 +408,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -424,11 +428,64 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)",
     "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "changeType": "UPSERT",
+    "aspectName": "chartKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "chartId": "myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)",
+    "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
             "viewLanguage": "m_query"
         }
     },
@@ -439,15 +496,20 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)"
+                },
+                {
+                    "id": "demo-workspace"
+                }
             ]
         }
     },
@@ -496,15 +558,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -515,14 +575,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
-            "viewLanguage": "m_query"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -532,52 +592,27 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "ownership",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)"
-                },
+            "owners": [
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)"
+                    "owner": "urn:li:corpuser:User1@foo.com",
+                    "type": "NONE"
                 },
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)"
+                    "owner": "urn:li:corpuser:User2@foo.com",
+                    "type": "NONE"
                 }
-            ]
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
         }
     },
     "systemMetadata": {
@@ -587,19 +622,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
-            "tags": []
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -610,13 +640,35 @@
 },
 {
     "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+            },
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)"
+                }
             ]
         }
     },
@@ -627,8 +679,8 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -643,8 +695,8 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
     "aspectName": "browsePaths",
     "aspect": {
@@ -661,15 +713,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "PowerBI Tile"
             ]
         }
     },
@@ -680,14 +731,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "username": "User1@foo.com"
         }
     },
     "systemMetadata": {
@@ -697,21 +747,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)"
-                },
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "username": "User2@foo.com"
         }
     },
     "systemMetadata": {
@@ -721,13 +763,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -737,16 +781,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -757,7 +799,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -773,7 +815,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -782,7 +824,7 @@
                 "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
+            "name": "ms_sql_native_table",
             "description": "hr pbi test description",
             "tags": []
         }
@@ -794,16 +836,67 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
     "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
-            "viewLanguage": "m_query"
-        }
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "demo-workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -819,8 +912,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -831,35 +923,21 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "hr pbi test description",
-            "tags": []
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)"
+                },
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -869,15 +947,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -890,24 +968,18 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "ownership",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "owners": [
+            "path": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
-                    "type": "NONE"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
-                    "type": "NONE"
+                    "id": "demo-workspace"
                 }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
+            ]
         }
     },
     "systemMetadata": {
@@ -920,11 +992,10 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -934,37 +1005,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)"
-                }
-            ]
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -974,13 +1027,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -990,15 +1045,15 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1008,15 +1063,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1026,14 +1081,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1043,67 +1097,20 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
-    "changeType": "PATCH",
-    "aspectName": "dashboardInfo",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
     "aspect": {
-        "json": [
-            {
-                "op": "add",
-                "path": "/customProperties/chartCount",
-                "value": "2"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceName",
-                "value": "demo-workspace"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C10-4684-8180-826122881108"
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "test_dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/description",
-                "value": "Description of test dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "value": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-                "value": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-            },
-            {
-                "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://localhost/dashboards/web/1"
+        "json": {
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
             },
-            {
-                "op": "add",
-                "path": "/lastModified",
-                "value": {
-                    "created": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "lastModified": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    }
-                }
-            }
-        ]
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
+        }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -1112,21 +1119,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)"
-                },
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1136,14 +1135,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -1154,21 +1153,13 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)"
-                },
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1178,8 +1169,8 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
index 66e87952bf141..0d3a0c0cc6f97 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
@@ -8,7 +8,10 @@
         "json": {
             "customProperties": {
                 "platform": "powerbi",
-                "workspace": "64ED5CAD-7C10-4684-8180-826122881108"
+                "workspace": "64ED5CAD-7C10-4684-8180-826122881108",
+                "workspace_id": "64ED5CAD-7C10-4684-8180-826122881108",
+                "workspace_name": "demo-workspace",
+                "workspace_type": "Workspace"
             },
             "name": "demo-workspace"
         }
@@ -86,15 +89,30 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
-            "viewLanguage": "m_query"
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
+            },
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": []
         }
     },
     "systemMetadata": {
@@ -104,30 +122,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "schemaMetadata",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "schemaName": "dbo_book_issue",
-            "platform": "urn:li:dataPlatform:powerbi",
-            "version": 0,
-            "created": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            },
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            },
-            "hash": "",
-            "platformSchema": {
-                "com.linkedin.schema.OtherSchema": {
-                    "rawSchema": ""
-                }
-            },
-            "fields": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -137,19 +138,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "",
-            "tags": []
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -159,13 +155,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "removed": false
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -175,16 +173,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "container": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
         }
     },
     "systemMetadata": {
@@ -194,20 +189,16 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "upstreams": [
+            "path": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.library.dbo.book_issue,PROD)",
-                    "type": "TRANSFORMED"
+                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
+                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
                 }
             ]
         }
@@ -222,10 +213,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -235,15 +228,38 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "tags": [
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+            },
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
                 {
-                    "tag": "urn:li:tag:Certified"
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)"
                 }
             ]
         }
@@ -258,19 +274,16 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
-                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
-                },
-                {
-                    "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
-                    "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
-                }
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -281,14 +294,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -299,12 +310,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "ms_sql_native_table",
+            "schemaName": "dbo_book_issue",
             "platform": "urn:li:dataPlatform:powerbi",
             "version": 0,
             "created": {
@@ -331,19 +342,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -354,12 +359,21 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "removed": false
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.library.dbo.book_issue,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -370,14 +384,15 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "tags": [
+                {
+                    "tag": "urn:li:tag:Certified"
+                }
             ]
         }
     },
@@ -388,21 +403,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.COMMOPSDB.dbo.V_PS_CD_RETENTION,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "PowerBI Tile"
             ]
         }
     },
@@ -416,10 +424,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -430,15 +440,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "tags": [
-                {
-                    "tag": "urn:li:tag:Certified"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -452,21 +460,18 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
-                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
-                },
-                {
-                    "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
-                    "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
-                }
-            ]
-        }
-    },
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "ms_sql_native_table",
+            "description": "",
+            "tags": []
+        }
+    },
     "systemMetadata": {
         "lastObserved": 1643871600000,
         "runId": "powerbi-test",
@@ -475,14 +480,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"database.sql.net\", \"analytics\", [Query=\"select * from analytics.sales_revenue\"])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -493,12 +496,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "revenue",
+            "schemaName": "ms_sql_native_table",
             "platform": "urn:li:dataPlatform:powerbi",
             "version": 0,
             "created": {
@@ -515,81 +518,7 @@
                     "rawSchema": ""
                 }
             },
-            "fields": [
-                {
-                    "fieldPath": "op_item_id",
-                    "nullable": false,
-                    "description": "op_item_id column description",
-                    "type": {
-                        "type": {
-                            "com.linkedin.schema.StringType": {}
-                        }
-                    },
-                    "nativeDataType": "String",
-                    "recursive": false,
-                    "isPartOfKey": false
-                },
-                {
-                    "fieldPath": "op_id",
-                    "nullable": false,
-                    "description": "op_id description",
-                    "type": {
-                        "type": {
-                            "com.linkedin.schema.StringType": {}
-                        }
-                    },
-                    "nativeDataType": "String",
-                    "recursive": false,
-                    "isPartOfKey": false
-                },
-                {
-                    "fieldPath": "op_product_name",
-                    "nullable": false,
-                    "type": {
-                        "type": {
-                            "com.linkedin.schema.StringType": {}
-                        }
-                    },
-                    "nativeDataType": "String",
-                    "recursive": false,
-                    "isPartOfKey": false
-                },
-                {
-                    "fieldPath": "event_name",
-                    "nullable": false,
-                    "description": "let\n x column description",
-                    "type": {
-                        "type": {
-                            "com.linkedin.schema.NullType": {}
-                        }
-                    },
-                    "nativeDataType": "measure",
-                    "recursive": false,
-                    "isPartOfKey": false
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "revenue",
-            "description": "",
-            "tags": []
+            "fields": []
         }
     },
     "systemMetadata": {
@@ -599,13 +528,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "removed": false
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -616,14 +546,20 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.COMMOPSDB.dbo.V_PS_CD_RETENTION,PROD)",
+                    "type": "TRANSFORMED"
+                }
             ]
         }
     },
@@ -635,19 +571,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "upstreams": [
+            "tags": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.analytics.analytics.sales_revenue,PROD)",
-                    "type": "TRANSFORMED"
+                    "tag": "urn:li:tag:Certified"
                 }
             ]
         }
@@ -660,7 +591,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -678,14 +609,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "tags": [
-                {
-                    "tag": "urn:li:tag:Certified"
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"database.sql.net\", \"analytics\", [Query=\"select * from analytics.sales_revenue\"])\nin\n    Source",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -696,20 +625,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
-                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
-                },
-                {
-                    "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
-                    "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -720,30 +642,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
             },
-            "inputs": []
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "revenue",
+            "description": "",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -753,8 +664,8 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -769,14 +680,82 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "schemaName": "revenue",
+            "platform": "urn:li:dataPlatform:powerbi",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.OtherSchema": {
+                    "rawSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "op_item_id",
+                    "nullable": false,
+                    "description": "op_item_id column description",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "String",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "op_id",
+                    "nullable": false,
+                    "description": "op_id description",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "String",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "op_product_name",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "String",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "event_name",
+                    "nullable": false,
+                    "description": "let\n x column description",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NullType": {}
+                        }
+                    },
+                    "nativeDataType": "measure",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -787,7 +766,7 @@
 },
 {
     "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "browsePaths",
     "aspect": {
@@ -804,13 +783,22 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "container": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.analytics.analytics.sales_revenue,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -820,16 +808,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "path": [
+            "tags": [
                 {
-                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
-                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
+                    "tag": "urn:li:tag:Certified"
                 }
             ]
         }
@@ -841,40 +828,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)"
-                }
-            ]
+            "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
         }
     },
     "systemMetadata": {
@@ -884,13 +844,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -900,14 +862,21 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
+            "path": [
+                {
+                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
+                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
+                },
+                {
+                    "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+                    "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
+                }
             ]
         }
     },
@@ -918,14 +887,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
         }
     },
     "systemMetadata": {
@@ -935,14 +903,46 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "path": [
+                {
+                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
+                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
+                },
+                {
+                    "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+                    "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
+                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
+                },
+                {
+                    "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+                    "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
+                }
             ]
         }
     },
@@ -1395,7 +1395,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset"
+                "Semantic Model"
             ]
         }
     },
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json
new file mode 100644
index 0000000000000..84d3d55defee0
--- /dev/null
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json
@@ -0,0 +1,332 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "powerbi",
+                "workspace": "Jane Smith Workspace",
+                "workspace_id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C",
+                "workspace_name": "Jane Smith Workspace",
+                "workspace_type": "PersonalGroup"
+            },
+            "name": "Jane Smith Workspace"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:powerbi"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PersonalGroup"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/Jane Smith Workspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "corpUserKey",
+    "aspect": {
+        "json": {
+            "username": "User1@foo.com"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "corpUserKey",
+    "aspect": {
+        "json": {
+            "username": "User2@foo.com"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "0"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "Jane Smith Workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:User1@foo.com",
+                    "type": "NONE"
+                },
+                {
+                    "owner": "urn:li:corpuser:User2@foo.com",
+                    "type": "NONE"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a",
+                    "urn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json
index ea1ee0df4b105..b2ec0f7a67a96 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json
@@ -17,6 +17,40 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.public_issue_history,DEV)",
@@ -40,14 +74,67 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.public_issue_history,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
     "aspect": {
-        "json": {
-            "removed": false
-        }
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "demo-workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -63,8 +150,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -75,15 +161,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -93,19 +177,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "SNOWFLAKE_TESTTABLE",
-            "description": "Library dataset description",
-            "tags": []
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -118,10 +197,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -131,15 +212,20 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)"
+                },
+                {
+                    "id": "demo-workspace"
+                }
             ]
         }
     },
@@ -151,14 +237,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -169,7 +253,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -178,7 +262,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query",
+            "name": "SNOWFLAKE_TESTTABLE",
             "description": "Library dataset description",
             "tags": []
         }
@@ -191,12 +275,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -209,13 +295,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -228,12 +313,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -265,13 +348,27 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.big-query-with-parameter,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "ownership",
     "aspect": {
         "json": {
-            "removed": false
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:User1@foo.com",
+                    "type": "NONE"
+                },
+                {
+                    "owner": "urn:li:corpuser:User2@foo.com",
+                    "type": "NONE"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
         }
     },
     "systemMetadata": {
@@ -288,8 +385,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -301,14 +397,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -319,7 +413,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -328,7 +422,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
+            "name": "snowflake native-query",
             "description": "Library dataset description",
             "tags": []
         }
@@ -341,12 +435,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -357,14 +453,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -378,12 +473,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -416,12 +509,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -438,8 +533,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -450,15 +544,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
-            "viewLanguage": "m_query"
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -468,32 +562,10 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
-            "tags": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "status",
     "aspect": {
         "json": {
             "removed": false
@@ -507,32 +579,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
             "viewLanguage": "m_query"
         }
     },
@@ -543,53 +596,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
-            "tags": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "PowerBI Tile"
             ]
         }
     },
@@ -601,14 +615,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -619,17 +631,17 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "hr pbi test description",
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
             "tags": []
         }
     },
@@ -640,13 +652,21 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)"
+                },
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -657,14 +677,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -675,29 +694,14 @@
     }
 },
 {
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
-    "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
-    "aspect": {
-        "json": {
-            "username": "User1@foo.com"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "corpUserKey",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "username": "User2@foo.com"
+            "dashboardTool": "powerbi",
+            "chartId": "aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -762,13 +766,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "removed": false
+            "username": "User1@foo.com"
         }
     },
     "systemMetadata": {
@@ -778,15 +782,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "corpUserKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "username": "User2@foo.com"
         }
     },
     "systemMetadata": {
@@ -796,14 +798,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -814,7 +817,7 @@
 },
 {
     "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
     "aspectName": "browsePaths",
     "aspect": {
@@ -831,21 +834,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)"
-                },
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -855,37 +850,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)"
-                }
-            ]
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -911,14 +888,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -949,11 +927,17 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)"
+                },
+                {
+                    "id": "demo-workspace"
+                }
             ]
         }
     },
@@ -967,16 +951,49 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "path": [
+            "dashboardTool": "powerbi",
+            "chartId": "aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "changeType": "UPSERT",
+    "aspectName": "chartInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+            },
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
                 {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)"
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)"
                 },
                 {
-                    "id": "demo-workspace"
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)"
                 }
             ]
         }
@@ -988,14 +1005,36 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -1006,67 +1045,16 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
-    "changeType": "PATCH",
-    "aspectName": "dashboardInfo",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
     "aspect": {
-        "json": [
-            {
-                "op": "add",
-                "path": "/customProperties/chartCount",
-                "value": "2"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceName",
-                "value": "demo-workspace"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C10-4684-8180-826122881108"
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "test_dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/description",
-                "value": "Description of test dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-                "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-            },
-            {
-                "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://localhost/dashboards/web/1"
-            },
-            {
-                "op": "add",
-                "path": "/lastModified",
-                "value": {
-                    "created": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "lastModified": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    }
-                }
-            }
-        ]
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
+        }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -1075,8 +1063,8 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1091,14 +1079,19 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "ms_sql_native_table",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1108,27 +1101,15 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "ownership",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
-                    "type": "NONE"
-                },
-                {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
-                    "type": "NONE"
-                }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -1138,21 +1119,31 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)"
-                },
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1163,7 +1154,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1179,7 +1170,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json b/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json
index 580a8d1a1db11..87e89e5ed74f2 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json
@@ -39,6 +39,40 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1645599600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1645599600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)",
@@ -48,8 +82,8 @@
         "json": {
             "timestampMillis": 1645599600000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "rowCount": 542300,
             "columnCount": 4,
@@ -107,41 +141,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1645599600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1645599600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)",
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json
index 094869bfd24f1..1805272b84d48 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json
@@ -63,8 +63,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -138,8 +137,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -213,8 +211,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -288,8 +285,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -363,8 +359,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -438,8 +433,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -513,8 +507,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -588,8 +581,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -663,8 +655,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -676,7 +667,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -692,7 +683,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -910,11 +901,12 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -927,12 +919,11 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -1108,11 +1099,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -1213,8 +1204,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1288,8 +1278,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1363,8 +1352,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1438,8 +1426,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1513,8 +1500,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1588,8 +1574,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1663,8 +1648,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -1676,7 +1660,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -1692,7 +1676,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -1904,11 +1888,11 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "typeNames": [
+                "PowerBI Page"
             ]
         }
     },
@@ -1922,11 +1906,11 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Page"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -2088,11 +2072,11 @@
         "json": {
             "owners": [
                 {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
+                    "owner": "urn:li:corpuser:User1@foo.com",
                     "type": "NONE"
                 },
                 {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
+                    "owner": "urn:li:corpuser:User2@foo.com",
                     "type": "NONE"
                 }
             ],
@@ -2129,9 +2113,660 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "dummy",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "public issue_history",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "SNOWFLAKE_TESTTABLE",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "big-query-with-parameter",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "Printable SalesMarketing"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Acryl sales marketing report"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/584cf13a-1485-41c2-a514-b1bb66fff163"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PaginatedReport"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2147,7 +2782,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json
index dcaa518a3c323..bc0feef4a127d 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json
@@ -17,6 +17,77 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "changeType": "UPSERT",
+    "aspectName": "chartInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
+            },
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
@@ -40,8 +111,8 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -63,8 +134,7 @@
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -75,15 +145,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
-            "viewLanguage": "m_query"
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -93,19 +163,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "SNOWFLAKE_TESTTABLE",
-            "description": "Library dataset description",
-            "tags": []
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -118,10 +183,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -131,15 +198,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -151,14 +217,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -169,7 +233,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -178,7 +242,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query",
+            "name": "SNOWFLAKE_TESTTABLE",
             "description": "Library dataset description",
             "tags": []
         }
@@ -193,10 +257,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -206,15 +272,16 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
             ]
         }
     },
@@ -226,14 +293,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -244,7 +309,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -253,7 +318,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
+            "name": "snowflake native-query",
             "description": "Library dataset description",
             "tags": []
         }
@@ -266,12 +331,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -282,14 +349,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -300,15 +366,37 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+            },
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -318,19 +406,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
-            "description": "Library dataset description",
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -341,12 +423,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -356,16 +440,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -376,14 +458,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -394,7 +474,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -403,7 +483,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "job-history",
+            "name": "big-query-with-parameter",
             "description": "Library dataset description",
             "tags": []
         }
@@ -415,13 +495,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -432,14 +514,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -450,15 +531,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
-            "viewLanguage": "m_query"
+            "paths": [
+                "/powerbi/demo-workspace"
+            ]
         }
     },
     "systemMetadata": {
@@ -468,19 +549,17 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
-            "description": "Library dataset description",
-            "tags": []
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -491,12 +570,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -506,15 +587,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -526,14 +606,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -544,17 +622,17 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "snowflake native-query-with-join",
+            "description": "Library dataset description",
             "tags": []
         }
     },
@@ -566,12 +644,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -584,13 +664,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -603,12 +682,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -639,14 +716,85 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "demo-workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -657,15 +805,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -675,52 +820,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
             },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
-                }
-            ]
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -730,13 +842,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -746,14 +860,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Tile"
+                "Table"
             ]
         }
     },
@@ -764,14 +878,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -781,15 +894,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -799,137 +911,8 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "chartInfo",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "chartKey",
-    "aspect": {
-        "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePaths",
-    "aspect": {
-        "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -949,13 +932,13 @@
 },
 {
     "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
     "aspectName": "browsePaths",
     "aspect": {
         "json": {
             "paths": [
-                "/powerbi/demo-workspace"
+                "/powerbi/second-demo-workspace"
             ]
         }
     },
@@ -967,7 +950,7 @@
 },
 {
     "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "PATCH",
     "aspectName": "dashboardInfo",
     "aspect": {
@@ -975,37 +958,22 @@
             {
                 "op": "add",
                 "path": "/customProperties/chartCount",
-                "value": "2"
+                "value": "0"
             },
             {
                 "op": "add",
                 "path": "/customProperties/workspaceName",
-                "value": "demo-workspace"
+                "value": "second-demo-workspace"
             },
             {
                 "op": "add",
                 "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+                "value": "64ED5CAD-7C22-4684-8180-826122881108"
             },
             {
                 "op": "add",
                 "path": "/title",
-                "value": "test_dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/description",
-                "value": "Description of test dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+                "value": "test_dashboard2"
             },
             {
                 "op": "add",
@@ -1036,7 +1004,7 @@
 },
 {
     "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1051,14 +1019,15 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1068,17 +1037,15 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1088,15 +1055,13 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/second-demo-workspace"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1106,52 +1071,20 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
-    "changeType": "PATCH",
-    "aspectName": "dashboardInfo",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
     "aspect": {
-        "json": [
-            {
-                "op": "add",
-                "path": "/customProperties/chartCount",
-                "value": "0"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceName",
-                "value": "second-demo-workspace"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C22-4684-8180-826122881108"
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "test_dashboard2"
-            },
-            {
-                "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://localhost/dashboards/web/1"
+        "json": {
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
-            {
-                "op": "add",
-                "path": "/lastModified",
-                "value": {
-                    "created": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "lastModified": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    }
-                }
-            }
-        ]
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
+        }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -1163,6 +1096,41 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
+    "aspectName": "dashboardKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
         "json": {
@@ -1176,14 +1144,19 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE"
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1211,5 +1184,23 @@
         "runId": "powerbi-test",
         "lastRunId": "no-run-id-provided"
     }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json
index bc5e844f679c7..bcf7b76285110 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json
@@ -17,6 +17,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
@@ -43,10 +59,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -56,15 +74,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -74,6 +91,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
@@ -115,13 +148,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "PowerBI Tile"
+            ]
         }
     },
     "systemMetadata": {
@@ -134,13 +169,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -150,20 +182,15 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "upstreams": [
+            "path": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_production_instance.pbi_test.test.testtable,PROD)",
-                    "type": "TRANSFORMED"
+                    "id": "demo-workspace"
                 }
             ]
         }
@@ -216,30 +243,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+                "Table"
             ]
         }
     },
@@ -253,19 +263,10 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,operations_analytics.transformed_prod.v_aps_sme_units_v4,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -275,15 +276,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
-            "viewLanguage": "m_query"
+            "dashboardTool": "powerbi",
+            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
         }
     },
     "systemMetadata": {
@@ -294,18 +294,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "big-query-with-parameter",
-            "description": "Library dataset description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -315,13 +311,37 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "createdFrom": "Dataset",
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
+            },
+            "title": "yearly_sales",
+            "description": "yearly_sales",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -332,14 +352,20 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_production_instance.pbi_test.test.testtable,PROD)",
+                    "type": "TRANSFORMED"
+                }
             ]
         }
     },
@@ -351,7 +377,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -362,7 +388,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-computing-dev-account.my-test-project.universal.d_wh_date,QA)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,operations_analytics.transformed_prod.v_aps_sme_units_v4,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -376,13 +402,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"",
             "viewLanguage": "m_query"
         }
     },
@@ -394,7 +420,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -403,7 +429,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "snowflake native-query-with-join",
+            "name": "big-query-with-parameter",
             "description": "Library dataset description",
             "tags": []
         }
@@ -416,7 +442,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -431,15 +457,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -451,28 +476,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD)",
-                    "type": "TRANSFORMED"
-                },
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_forecast,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -483,15 +493,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
-            "viewLanguage": "m_query"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -501,35 +509,52 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+                "createdFrom": "Dataset",
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
             },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "job-history",
-            "description": "Library dataset description",
-            "tags": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
+            "title": "test_tile",
+            "description": "test_tile",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
+                },
+                {
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -540,14 +565,20 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-computing-dev-account.my-test-project.universal.d_wh_date,QA)",
+                    "type": "TRANSFORMED"
+                }
             ]
         }
     },
@@ -558,21 +589,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle-sales-instance.salesdb.hr.employees,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "PowerBI Tile"
             ]
         }
     },
@@ -584,13 +608,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "viewProperties",
     "aspect": {
         "json": {
             "materialized": false,
-            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
             "viewLanguage": "m_query"
         }
     },
@@ -602,7 +626,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -611,7 +635,7 @@
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
             },
             "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
-            "name": "postgres_test_table",
+            "name": "snowflake native-query-with-join",
             "description": "Library dataset description",
             "tags": []
         }
@@ -623,13 +647,17 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "demo-workspace"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -640,15 +668,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -658,22 +683,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "chartKey",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "dashboardTool": "powerbi",
+            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
         }
     },
     "systemMetadata": {
@@ -702,18 +719,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "dbo_book_issue",
-            "description": "hr pbi test description",
-            "tags": []
+            "materialized": false,
+            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -724,12 +737,18 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "ms_sql_native_table",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -742,13 +761,16 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
-            ]
+            "customProperties": {
+                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
+            "name": "dbo_book_issue",
+            "description": "hr pbi test description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -759,21 +781,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,library.dbo.book_issue,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -784,14 +797,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\"",
-            "viewLanguage": "m_query"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -804,16 +817,12 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
-            },
-            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details",
-            "name": "ms_sql_native_table",
-            "description": "hr pbi test description",
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -824,7 +833,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -839,15 +848,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "browsePaths",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Dataset Table",
-                "View"
+            "paths": [
+                "/powerbi/demo-workspace"
             ]
         }
     },
@@ -884,7 +892,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -900,7 +908,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "corpUserKey",
     "aspect": {
@@ -915,51 +923,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details"
-            },
-            "title": "test_tile",
-            "description": "test_tile",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -970,14 +941,67 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
     "aspect": {
-        "json": {
-            "removed": false
-        }
+        "json": [
+            {
+                "op": "add",
+                "path": "/customProperties/chartCount",
+                "value": "2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceName",
+                "value": "demo-workspace"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/workspaceId",
+                "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -986,14 +1010,29 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_forecast,PROD)",
+                    "type": "TRANSFORMED"
+                }
             ]
         }
     },
@@ -1004,14 +1043,22 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,library.dbo.book_issue,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1021,15 +1068,13 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1039,17 +1084,14 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "dashboardKey",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
         }
     },
     "systemMetadata": {
@@ -1059,35 +1101,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "chartInfo",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "createdFrom": "Dataset",
-                "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details"
-            },
-            "title": "yearly_sales",
-            "description": "yearly_sales",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
-                },
+            "path": [
                 {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)"
+                    "id": "demo-workspace"
                 }
             ]
         }
@@ -1099,13 +1121,27 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "ownership",
     "aspect": {
         "json": {
-            "removed": false
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:User1@foo.com",
+                    "type": "NONE"
+                },
+                {
+                    "owner": "urn:li:corpuser:User2@foo.com",
+                    "type": "NONE"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
         }
     },
     "systemMetadata": {
@@ -1115,14 +1151,22 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "chartKey",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385"
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle-sales-instance.salesdb.hr.employees,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1132,15 +1176,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "PowerBI Tile"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1150,15 +1194,15 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
-            ]
+            "materialized": false,
+            "viewLogic": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
@@ -1168,17 +1212,19 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "demo-workspace"
-                }
-            ]
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "postgres_test_table",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1188,14 +1234,14 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePaths",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "paths": [
-                "/powerbi/demo-workspace"
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -1206,67 +1252,14 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
-    "changeType": "PATCH",
-    "aspectName": "dashboardInfo",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
     "aspect": {
-        "json": [
-            {
-                "op": "add",
-                "path": "/customProperties/chartCount",
-                "value": "2"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceName",
-                "value": "demo-workspace"
-            },
-            {
-                "op": "add",
-                "path": "/customProperties/workspaceId",
-                "value": "64ED5CAD-7C10-4684-8180-826122881108"
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "test_dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/description",
-                "value": "Description of test dashboard"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
-            },
-            {
-                "op": "add",
-                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
-                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-            },
-            {
-                "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://localhost/dashboards/web/1"
-            },
-            {
-                "op": "add",
-                "path": "/lastModified",
-                "value": {
-                    "created": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "lastModified": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    }
-                }
-            }
-        ]
+        "json": {
+            "removed": false
+        }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
@@ -1275,13 +1268,19 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details",
+            "name": "job-history",
+            "description": "Library dataset description",
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -1291,14 +1290,15 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardKey",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "dashboardTool": "powerbi",
-            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -1308,27 +1308,13 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "ownership",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:users.User1@foo.com",
-                    "type": "NONE"
-                },
-                {
-                    "owner": "urn:li:corpuser:users.User2@foo.com",
-                    "type": "NONE"
-                }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -1338,15 +1324,20 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
+            "upstreams": [
                 {
-                    "id": "demo-workspace"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)",
+                    "type": "TRANSFORMED"
                 }
             ]
         }
@@ -1359,7 +1350,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User1@foo.com",
+    "entityUrn": "urn:li:corpuser:User1@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1375,7 +1366,7 @@
 },
 {
     "entityType": "corpuser",
-    "entityUrn": "urn:li:corpuser:users.User2@foo.com",
+    "entityUrn": "urn:li:corpuser:User2@foo.com",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/powerbi/mock_data/cross_workspace_mock_response.json b/metadata-ingestion/tests/integration/powerbi/mock_data/cross_workspace_mock_response.json
new file mode 100644
index 0000000000000..9b6931727bc1a
--- /dev/null
+++ b/metadata-ingestion/tests/integration/powerbi/mock_data/cross_workspace_mock_response.json
@@ -0,0 +1,220 @@
+{
+    "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "A8D655A6-F521-477E-8C22-255018583BF4",
+                    "isReadOnly": true,
+                    "name": "Sales",
+                    "type": "Workspace",
+                    "state": "Active"
+                },
+                {
+                    "id": "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492",
+                    "isReadOnly": true,
+                    "name": "Global Workspace",
+                    "type": "Workspace",
+                    "state": "Active"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": []
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/A8D655A6-F521-477E-8C22-255018583BF4/dashboards": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "value": [
+                    {
+                        "id": "A1C7204F-4D04-4E5E-B886-B30EA2C64CB3",
+                        "isReadOnly": true,
+                        "displayName": "test_dashboard",
+                        "description": "Description of test dashboard",
+                        "embedUrl": "https://localhost/dashboards/embed/1",
+                        "webUrl": "https://localhost/dashboards/web/1"
+                    }
+                ]
+            }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/dashboards": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "value": []
+            }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/A8D655A6-F521-477E-8C22-255018583BF4/dashboards/A1C7204F-4D04-4E5E-B886-B30EA2C64CB3/tiles": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "885D1762-1655-46BA-AFE3-74C6EC403A9E",
+                    "title": "Sale Order Tile",
+                    "embedUrl": "https://localhost/tiles/embed/1",
+                    "datasetId": "317456E5-1FC7-4BDC-9C84-1185825E293D"
+                },
+                {
+                    "id": "945C2C2A-4588-45DE-8385-F24F5E39A57C",
+                    "title": "Yearly Sales",
+                    "embedUrl": "https://localhost/tiles/embed/2",
+                    "datasetId": "FE362B98-956E-4394-BA37-6367EE6435E9"
+                },
+                {
+                    "id": "B847EBDA-BC48-4F92-8E16-6B46D900E7BB",
+                    "title": "Not Present In Current Ingestion",
+                    "embedUrl": "https://localhost/tiles/embed/2",
+                    "datasetId": "0F0ADA0E-E38A-44F6-B667-90E93A96F5A1"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/62DAF926-0B18-4FF1-982C-2A3EB6B8F0E4": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "workspaces": [
+                {
+                    "id": "A8D655A6-F521-477E-8C22-255018583BF4",
+                    "name": "Sales",
+                    "type": "Workspace",
+                    "state": "Active",
+                    "datasets": [
+                        {
+                            "id": "317456E5-1FC7-4BDC-9C84-1185825E293D",
+                            "endorsementDetails": {
+                                "endorsement": "Promoted"
+                            },
+                            "name": "sales semantic model",
+                            "tables": [
+                                {
+                                    "name": "public issue_history",
+                                    "source": [
+                                        {
+                                            "expression": "dummy"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                },
+                {
+                    "id": "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492",
+                    "name": "Global Workspace",
+                    "type": "Workspace",
+                    "state": "Active",
+                    "datasets": [
+                        {
+                            "id": "FE362B98-956E-4394-BA37-6367EE6435E9",
+                            "endorsementDetails": {
+                                "endorsement": "Promoted"
+                            },
+                            "name": "base_records",
+                            "tables": [
+                                {
+                                    "name": "core_sales_set",
+                                    "source": [
+                                        {
+                                            "expression": "dummy"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/62DAF926-0B18-4FF1-982C-2A3EB6B8F0E4": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "status": "SUCCEEDED"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/81B02907-E2A3-45C3-B505-3781839C8CAA": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "workspaces": [
+                {
+                    "id": "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492",
+                    "name": "Global Workspace",
+                    "type": "Workspace",
+                    "state": "Active",
+                    "datasets": [
+                        {
+                            "id": "FE362B98-956E-4394-BA37-6367EE6435E9",
+                            "endorsementDetails": {
+                                "endorsement": "Promoted"
+                            },
+                            "name": "base_records",
+                            "tables": [
+                                {
+                                    "name": "core_sales_set",
+                                    "source": [
+                                        {
+                                            "expression": "dummy"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/81B02907-E2A3-45C3-B505-3781839C8CAA": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "status": "SUCCEEDED"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/A8D655A6-F521-477E-8C22-255018583BF4/datasets/317456E5-1FC7-4BDC-9C84-1185825E293D": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "id": "317456E5-1FC7-4BDC-9C84-1185825E293D",
+            "name": "sales semantic model",
+            "description": "sales semantic model",
+            "webUrl": "http://localhost/groups/A8D655A6-F521-477E-8C22-255018583BF4/datasets/317456E5-1FC7-4BDC-9C84-1185825E293D"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/datasets/FE362B98-956E-4394-BA37-6367EE6435E9": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "id": "FE362B98-956E-4394-BA37-6367EE6435E9",
+            "name": "base_records",
+            "description": "base_records",
+            "webUrl": "http://localhost/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/datasets/FE362B98-956E-4394-BA37-6367EE6435E9"
+        }
+    }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/mock_data/default_mock_response.json b/metadata-ingestion/tests/integration/powerbi/mock_data/default_mock_response.json
new file mode 100644
index 0000000000000..28972fbded9e8
--- /dev/null
+++ b/metadata-ingestion/tests/integration/powerbi/mock_data/default_mock_response.json
@@ -0,0 +1,558 @@
+{
+    "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "64ED5CAD-7C10-4684-8180-826122881108",
+                    "isReadOnly": true,
+                    "name": "demo-workspace",
+                    "type": "Workspace",
+                    "state": "Active"
+
+                },
+                {
+                    "id": "64ED5CAD-7C22-4684-8180-826122881108",
+                    "isReadOnly": true,
+                    "name": "second-demo-workspace",
+                    "type": "Workspace",
+                    "state": "Active"
+
+                },
+                {
+                    "id": "64ED5CAD-7322-4684-8180-826122881108",
+                    "isReadOnly": true,
+                    "name": "Workspace 2",
+                    "type": "Workspace",
+                    "state": "Active"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": []
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE",
+                    "isReadOnly": true,
+                    "displayName": "test_dashboard",
+                    "description": "Description of test dashboard",
+                    "embedUrl": "https://localhost/dashboards/embed/1",
+                    "webUrl": "https://localhost/dashboards/web/1"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE",
+                    "isReadOnly": true,
+                    "displayName": "test_dashboard2",
+                    "embedUrl": "https://localhost/dashboards/embed/1",
+                    "webUrl": "https://localhost/dashboards/web/1"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/reports/5b218778-e7a5-4d73-8187-f10824047715/users": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "identifier": "User1@foo.com",
+                    "displayName": "user1",
+                    "emailAddress": "User1@foo.com",
+                    "datasetUserAccessRight": "ReadWrite",
+                    "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
+                    "principalType": "User"
+                },
+                {
+                    "identifier": "User2@foo.com",
+                    "displayName": "user2",
+                    "emailAddress": "User2@foo.com",
+                    "datasetUserAccessRight": "ReadWrite",
+                    "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
+                    "principalType": "User"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/users": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "identifier": "User1@foo.com",
+                    "displayName": "user1",
+                    "emailAddress": "User1@foo.com",
+                    "datasetUserAccessRight": "ReadWrite",
+                    "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
+                    "principalType": "User"
+                },
+                {
+                    "identifier": "User2@foo.com",
+                    "displayName": "user2",
+                    "emailAddress": "User2@foo.com",
+                    "datasetUserAccessRight": "ReadWrite",
+                    "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
+                    "principalType": "User"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/users": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "identifier": "User3@foo.com",
+                    "displayName": "user3",
+                    "emailAddress": "User3@foo.com",
+                    "datasetUserAccessRight": "ReadWrite",
+                    "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
+                    "principalType": "User"
+                },
+                {
+                    "identifier": "User4@foo.com",
+                    "displayName": "user4",
+                    "emailAddress": "User4@foo.com",
+                    "datasetUserAccessRight": "ReadWrite",
+                    "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
+                    "principalType": "User"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0",
+                    "title": "test_tile",
+                    "embedUrl": "https://localhost/tiles/embed/1",
+                    "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445"
+                },
+                {
+                    "id": "23212598-23b5-4980-87cc-5fc0ecd84385",
+                    "title": "yearly_sales",
+                    "embedUrl": "https://localhost/tiles/embed/2",
+                    "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/tiles": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": []
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "id": "05169CD2-E713-41E6-9600-1D8066D95445",
+            "name": "library-dataset",
+            "description": "Library dataset description",
+            "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "id": "05169CD2-E713-41E6-96AA-1D8066D95445",
+            "name": "library-dataset",
+            "description": "Library dataset description",
+            "webUrl": "http://localhost/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+            "name": "hr_pbi_test",
+            "description": "hr pbi test description",
+            "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/datasources": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
+                    "datasourceType": "PostgreSql",
+                    "connectionDetails": {
+                        "database": "library_db",
+                        "server": "foo"
+                    }
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445/datasources": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
+                    "datasourceType": "PostgreSql",
+                    "connectionDetails": {
+                        "database": "library_db",
+                        "server": "foo"
+                    }
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4674efd1-603c-4129-8d82-03cf2be05aff": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "status": "SUCCEEDED"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/a674efd1-603c-4129-8d82-03cf2be05aff": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "status": "SUCCEEDED"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "workspaces": [
+                {
+                    "id": "64ED5CAD-7C10-4684-8180-826122881108",
+                    "name": "demo-workspace",
+                    "type": "Workspace",
+                    "state": "Active",
+                    "datasets": [
+                        {
+                            "id": "05169CD2-E713-41E6-9600-1D8066D95445",
+                            "endorsementDetails": {
+                                "endorsement": "Promoted"
+                            },
+                            "name": "test_sf_pbi_test",
+                            "tables": [
+                                {
+                                    "name": "public issue_history",
+                                    "source": [
+                                        {
+                                            "expression": "dummy"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                },
+                                {
+                                    "name": "SNOWFLAKE_TESTTABLE",
+                                    "source": [
+                                        {
+                                            "expression": "let\n    Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n    PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n    TESTTABLE_Table"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                },
+                                {
+                                    "name": "snowflake native-query",
+                                    "source": [
+                                        {
+                                            "expression": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n    #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n    #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n    #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n    #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n    then [UNIT] * 361\nelse 0),\n    #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n    #\"Added Custom2\""
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                },
+                                {
+                                    "name": "big-query-with-parameter",
+                                    "source": [
+                                        {
+                                            "expression": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\""
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                },
+                                {
+                                    "name": "snowflake native-query-with-join",
+                                    "source": [
+                                        {
+                                            "expression": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                },
+                                {
+                                    "name": "job-history",
+                                    "source": [
+                                        {
+                                            "expression": "let\n    Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                },
+                                {
+                                    "name": "postgres_test_table",
+                                    "source": [
+                                        {
+                                            "expression": "let\n    Source = PostgreSQL.Database(\"localhost\"  ,   \"mics\"      ),\n  public_order_date =    Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                }
+                            ]
+                        },
+                        {
+                            "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
+                            "name": "hr_pbi_test",
+                            "tables": [
+                                {
+                                    "name": "dbo_book_issue",
+                                    "source": [
+                                        {
+                                            "expression": "let\n    Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue"
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                },
+                                {
+                                    "name": "ms_sql_native_table",
+                                    "source": [
+                                        {
+                                            "expression": "let\n    Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n    #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n    #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #\"Added Custom1\""
+                                        }
+                                    ],
+                                    "datasourceUsages": [
+                                        {
+                                            "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3"
+                                        }
+                                    ]
+                                }
+                            ]
+                        },
+                        {
+                            "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
+                            "tables": [
+                                {
+                                    "name": "employee_ctc",
+                                    "source": [
+                                        {
+                                            "expression": "dummy"
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ],
+                    "dashboards": [
+                        {
+                            "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE",
+                            "isReadOnly": true
+                        }
+                    ],
+                    "reports": [
+                        {
+                            "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                            "reportType": "PaginatedReport",
+                            "id": "5b218778-e7a5-4d73-8187-f10824047715",
+                            "name": "SalesMarketing",
+                            "description": "Acryl sales marketing report"
+                        }
+                    ]
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/a674efd1-603c-4129-8d82-03cf2be05aff": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "workspaces": [
+                {
+                    "id": "64ED5CAD-7C22-4684-8180-826122881108",
+                    "name": "second-demo-workspace",
+                    "type": "Workspace",
+                    "state": "Active",
+                    "datasets": [
+                        {
+                            "id": "05169CD2-E713-41E6-96AA-1D8066D95445",
+                            "tables": [
+                                {
+                                    "name": "public articles",
+                                    "source": [
+                                        {
+                                            "expression": "dummy"
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ],
+                    "dashboards": [
+                        {
+                            "id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE",
+                            "isReadOnly": true
+                        }
+                    ],
+                    "reports": [
+                        {
+                            "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                            "id": "5b218778-e7a5-4d73-8187-f10824047715",
+                            "reportType": "PaginatedReport",
+                            "name": "SalesMarketing",
+                            "description": "Acryl sales marketing report"
+                        }
+                    ]
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                    "id": "5b218778-e7a5-4d73-8187-f10824047715",
+                    "reportType": "PowerBIReport",
+                    "name": "SalesMarketing",
+                    "description": "Acryl sales marketing report",
+                    "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715",
+                    "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48"
+                },
+                {
+                    "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+                    "id": "584cf13a-1485-41c2-a514-b1bb66fff163",
+                    "reportType": "PaginatedReport",
+                    "name": "Printable SalesMarketing",
+                    "description": "Acryl sales marketing report",
+                    "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163",
+                    "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=584cf13a-1485-41c2-a514-b1bb66fff163&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
+            "id": "5b218778-e7a5-4d73-8187-f10824047715",
+            "reportType": "PaginatedReport",
+            "name": "SalesMarketing",
+            "description": "Acryl sales marketing report",
+            "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715",
+            "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "displayName": "Regional Sales Analysis",
+                    "name": "ReportSection",
+                    "order": "0"
+                },
+                {
+                    "displayName": "Geographic Analysis",
+                    "name": "ReportSection1",
+                    "order": "1"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/parameters": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "name": "Parameter - Source",
+                    "type": "Text",
+                    "isRequired": true,
+                    "currentValue": "my-test-project"
+                },
+                {
+                    "name": "My bq project",
+                    "type": "Text",
+                    "isRequired": true,
+                    "currentValue": "gcp_billing"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
+            "name": "employee-dataset",
+            "description": "Employee Management",
+            "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/584cf13a-1485-41c2-a514-b1bb66fff163/pages": {
+        "method": "GET",
+        "status_code": 400,
+        "text": "{\"error\":{\"code\":\"InvalidRequest\",\"message\":\"Request is currently not supported for RDL reports\"}}"
+    }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_type_filter.json b/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_type_filter.json
new file mode 100644
index 0000000000000..9a6ea36e7ab46
--- /dev/null
+++ b/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_type_filter.json
@@ -0,0 +1,76 @@
+{
+    "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C",
+                    "isReadOnly": true,
+                    "name": "Jane Smith Workspace",
+                    "type": "PersonalGroup",
+                    "state": "Active"
+                },
+                {
+                    "id": "C6B5DBBC-7580-406C-A6BE-72628C28801C",
+                    "isReadOnly": true,
+                    "name": "Sales",
+                    "type": "Workspace",
+                    "state": "Active"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": []
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "workspaces": [
+                {
+                    "id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C",
+                    "name": "Jane Smith Workspace",
+                    "type": "PersonalGroup",
+                    "state": "Active",
+                    "datasets": []
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/90E9E256-3D6D-4D38-86C8-6CCCBD8C170C/dashboards": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE",
+                    "isReadOnly": true,
+                    "displayName": "test_dashboard",
+                    "description": "Description of test dashboard",
+                    "embedUrl": "https://localhost/dashboards/embed/1",
+                    "webUrl": "https://localhost/dashboards/web/1"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "status": "SUCCEEDED"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/90E9E256-3D6D-4D38-86C8-6CCCBD8C170C/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "value": []
+            }
+    }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_with_app_mock_response.json b/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_with_app_mock_response.json
new file mode 100644
index 0000000000000..712982741ccaa
--- /dev/null
+++ b/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_with_app_mock_response.json
@@ -0,0 +1,149 @@
+{
+    "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "8F756DE6-26AD-45FF-A201-44276FF1F561",
+                    "isReadOnly": true,
+                    "name": "Workspace For App Testing",
+                    "type": "Workspace",
+                    "state": "Active"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": []
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/6147FCEB-7531-4449-8FB6-1F7A5431BF2D": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "workspaces": [
+                {
+                    "id": "8F756DE6-26AD-45FF-A201-44276FF1F561",
+                    "name": "Workspace For App Testing",
+                    "type": "Workspace",
+                    "state": "Active",
+                     "reports": [
+                      {
+                       "reportType": "PowerBIReport",
+                       "id": "455AB99B-E110-46E6-90D3-F015CABD1156",
+                       "name": "GitHub Progress",
+                       "datasetId": "2F99BE64-673D-4DA8-BF4F-02629A1F2C8F",
+                       "createdDateTime": "2024-10-01T06:27:00.51",
+                       "modifiedDateTime": "2024-10-01T06:27:00.51",
+                       "modifiedBy": "abc@fake.com",
+                       "createdBy": "abc@fake.com",
+                       "modifiedById": "97ABB057-CB8B-480D-AA1C-B3E7F0A16EC5",
+                       "createdById": "97ABB057-CB8B-480D-AA1C-B3E7F0A16EC5",
+                       "datasetWorkspaceId": "8F756DE6-26AD-45FF-A201-44276FF1F561",
+                       "users": [
+                        {
+                         "reportUserAccessRight": "Owner",
+                         "emailAddress": "abc@fake.com",
+                         "displayName": "John Smith",
+                         "identifier": "abc@fake.com",
+                         "graphId": "97ABB057-CB8B-480D-AA1C-B3E7F0A16EC5",
+                         "principalType": "User",
+                         "userType": "Member"
+                        }
+                       ]
+                      },
+                      {
+                       "reportType": "PowerBIReport",
+                       "id": "ecc35189-e67a-4d8a-9037-403317bd5808",
+                       "name": "[App] GitHub Progress",
+                       "datasetId": "2F99BE64-673D-4DA8-BF4F-02629A1F2C8F",
+                       "appId": "2A4D0E82-E7A4-45B1-BD72-2A2CF82C9CB6",
+                       "createdDateTime": "2024-10-01T06:29:34.75",
+                       "modifiedDateTime": "2024-10-01T07:22:55.397",
+                       "originalReportObjectId": "455AB99B-E110-46E6-90D3-F015CABD1156",
+                       "modifiedBy": "abc@fake.com",
+                       "createdBy": "abc@fake.com",
+                       "modifiedById": "97ABB057-CB8B-480D-AA1C-B3E7F0A16EC5",
+                       "createdById": "97ABB057-CB8B-480D-AA1C-B3E7F0A16EC5",
+                       "datasetWorkspaceId": "8F756DE6-26AD-45FF-A201-44276FF1F561"
+                      }
+                     ],
+                     "dashboards": [
+                       {
+                         "id": "744B07E3-FAA7-4BD7-BD17-3220BF0F6301",
+                         "displayName": "Pet Overview",
+                         "isReadOnly": false,
+                         "tiles": [
+                         ]
+                       },
+                       {
+                         "id": "DB85B962-74BA-4821-900B-59AC5F70AADE",
+                         "appId": "2A4D0E82-E7A4-45B1-BD72-2A2CF82C9CB6",
+                         "displayName": "[App] Pet Overview",
+                         "isReadOnly": false,
+                         "tiles": [
+                         ]
+                       }
+                     ],
+                    "datasets": []
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/8F756DE6-26AD-45FF-A201-44276FF1F561/dashboards": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                    "id": "A700E2C1-D008-42DF-AFCA-A70A87D0B2A3",
+                    "isReadOnly": true,
+                    "displayName": "test_dashboard",
+                    "description": "Description of test dashboard",
+                    "embedUrl": "https://localhost/dashboards/embed/1",
+                    "webUrl": "https://localhost/dashboards/web/1"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/6147FCEB-7531-4449-8FB6-1F7A5431BF2D": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "status": "SUCCEEDED"
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/groups/8F756DE6-26AD-45FF-A201-44276FF1F561/dashboards/A700E2C1-D008-42DF-AFCA-A70A87D0B2A3/tiles": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "value": []
+            }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/apps?%24skip=0&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": [
+                {
+                  "id": "2A4D0E82-E7A4-45B1-BD72-2A2CF82C9CB6",
+                  "description": "The finance app",
+                  "name": "Finance",
+                  "publishedBy": "Bill",
+                  "lastUpdate": "2024-09-26T04:20:34.513Z"
+                }
+            ]
+        }
+    },
+    "https://api.powerbi.com/v1.0/myorg/admin/apps?%24skip=1000&%24top=1000": {
+        "method": "GET",
+        "status_code": 200,
+        "json": {
+            "value": []
+        }
+    }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py
index 91c6082524389..b636c12cfda06 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py
@@ -58,21 +58,28 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
             "status_code": 200,
             "json": admin_datasets_response,
         },
-        "https://api.powerbi.com/v1.0/myorg/admin/groups": {
+        "https://api.powerbi.com/v1.0/myorg/admin/groups?%24skip=0&%24top=1000": {
             "method": "GET",
             "status_code": 200,
             "json": {
-                "@odata.count": 3,
                 "value": [
                     {
                         "id": "64ED5CAD-7C10-4684-8180-826122881108",
                         "isReadOnly": True,
                         "name": "demo-workspace",
                         "type": "Workspace",
+                        "state": "Active",
                     }
                 ],
             },
         },
+        "https://api.powerbi.com/v1.0/myorg/admin/groups?%24skip=1000&%24top=1000": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "value": [],
+            },
+        },
         "https://api.powerbi.com/v1.0/myorg/admin/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": {
             "method": "GET",
             "status_code": 200,
@@ -220,6 +227,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
                     {
                         "id": "64ED5CAD-7C10-4684-8180-826122881108",
                         "name": "demo-workspace",
+                        "type": "Workspace",
                         "state": "Active",
                         "datasets": [
                             {
@@ -391,6 +399,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
                             {
                                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
                                 "id": "5b218778-e7a5-4d73-8187-f10824047715",
+                                "reportType": "PowerBIReport",
                                 "name": "SalesMarketing",
                                 "description": "Acryl sales marketing report",
                             }
@@ -422,6 +431,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
                         "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
                         "id": "5b218778-e7a5-4d73-8187-f10824047715",
                         "name": "SalesMarketing",
+                        "reportType": "PowerBIReport",
                         "description": "Acryl sales marketing report",
                         "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715",
                         "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48",
@@ -436,6 +446,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
                 "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
                 "id": "5b218778-e7a5-4d73-8187-f10824047715",
                 "name": "SalesMarketing",
+                "reportType": "PowerBIReport",
                 "description": "Acryl sales marketing report",
                 "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715",
                 "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48",
diff --git a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
index aac8f04c1c030..832d00d9c5470 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
@@ -1,12 +1,16 @@
 import logging
 import sys
+import time
 from typing import List, Tuple
+from unittest.mock import MagicMock, patch
 
 import pytest
 from lark import Tree
 
+import datahub.ingestion.source.powerbi.m_query.data_classes
 import datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes as powerbi_data_classes
 from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.source import StructuredLogLevel
 from datahub.ingestion.source.powerbi.config import (
     PowerBiDashboardSourceConfig,
     PowerBiDashboardSourceReport,
@@ -15,8 +19,11 @@
     AbstractDataPlatformInstanceResolver,
     create_dataplatform_instance_resolver,
 )
-from datahub.ingestion.source.powerbi.m_query import parser, resolver, tree_function
-from datahub.ingestion.source.powerbi.m_query.resolver import DataPlatformTable, Lineage
+from datahub.ingestion.source.powerbi.m_query import parser, tree_function
+from datahub.ingestion.source.powerbi.m_query.data_classes import (
+    DataPlatformTable,
+    Lineage,
+)
 
 pytestmark = pytest.mark.integration_batch_2
 
@@ -51,9 +58,40 @@
     'let\n Source = DatabricksMultiCloud.Catalogs("abc.cloud.databricks.com", "/sql/gh2cfe3fe1d4c7cd", [Catalog="data_analysis", Database="summary", EnableAutomaticProxyDiscovery=null]),\n vips_data_summary_dev = Source{[Item="vips_data",Schema="summary",Catalog="data_analysis"]}[Data],\n #"Changed Type" = Table.TransformColumnTypes(vips_data_summary_dev,{{"vipstartDate", type date}, {"enteredDate", type datetime}, {"estDraftDate", type datetime}, {"estPublDate", type datetime}})\nin\n #"Changed Type"',
     'let\n Source = Value.NativeQuery(Snowflake.Databases("0DD93C6BD5A6.snowflakecomputing.com","sales_analytics_warehouse_prod",[Role="sales_analytics_member_ad"]){[Name="ORDERING"]}[Data], "SELECT#(lf) DISTINCT#(lf) T5.PRESENTMENT_START_DATE#(lf),T5.PRESENTMENT_END_DATE#(lf),T5.DISPLAY_NAME#(lf),T5.NAME#(tab)#(lf),T5.PROMO_DISPLAY_NAME#(lf),T5.REGION#(lf),T5.ID#(lf),T5.WALKOUT#(lf),T6.DEAL_ID#(lf),T6.TYPE#(lf),T5.FREE_PERIOD#(lf),T6.PRICE_MODIFICATION#(lf)#(lf)FROM#(lf)#(lf)(#(lf)    SELECT #(lf) T1.NAME#(lf),DATE(T1.CREATED_AT) as CREATED_AT#(lf),T1.PROMO_CODE#(lf),T1.STATUS#(lf),DATE(T1.UPDATED_AT) as UPDATED_AT#(lf),T1.ID#(lf),T1.DISPLAY_NAME as PROMO_DISPLAY_NAME#(lf),T4.*#(lf)FROM#(lf)(SELECT#(lf) DISTINCT#(lf) NAME#(lf),CREATED_AT#(lf),PROMO_CODE#(lf),STATUS#(lf),UPDATED_AT#(lf),ID#(lf),DISPLAY_NAME#(lf) FROM RAW.PROMOTIONS#(lf)#(lf)) T1#(lf)INNER JOIN#(lf)#(lf) (#(lf)    SELECT #(lf) T3.PRODUCT_STATUS#(lf),T3.CODE#(lf),T3.REGION#(lf),T3.DISPLAY_ORDER_SEQUENCE#(lf),T3.PRODUCT_LINE_ID#(lf),T3.DISPLAY_NAME#(lf),T3.PRODUCT_TYPE#(lf),T3.ID as PROD_TBL_ID#(lf),T3.NAME as PROD_TBL_NAME#(lf),DATE(T2.PRESENTMENT_END_DATE) as PRESENTMENT_END_DATE#(lf),T2.PRICE_COMMITMENT_PERIOD#(lf),T2.NAME as SEAL_TBL_NAME#(lf),DATE(T2.CREATED_AT) as SEAL_TBL_CREATED_AT#(lf),T2.DESCRIPTION#(lf),T2.FREE_PERIOD#(lf),T2.WALKOUT#(lf),T2.PRODUCT_CAT_ID#(lf),T2.PROMOTION_ID#(lf),DATE(T2.PRESENTMENT_START_DATE) as PRESENTMENT_START_DATE#(lf),YEAR(T2.PRESENTMENT_START_DATE) as DEAL_YEAR_START#(lf),MONTH(T2.PRESENTMENT_START_DATE) as DEAL_MONTH_START#(lf),T2.DEAL_TYPE#(lf),DATE(T2.UPDATED_AT) as SEAL_TBL_UPDATED_AT#(lf),T2.ID as SEAL_TBL_ID#(lf),T2.STATUS as SEAL_TBL_STATUS#(lf)FROM#(lf)(SELECT#(lf) DISTINCT#(lf) PRODUCT_STATUS#(lf),CODE#(lf),REGION#(lf),DISPLAY_ORDER_SEQUENCE#(lf),PRODUCT_LINE_ID#(lf),DISPLAY_NAME#(lf),PRODUCT_TYPE#(lf),ID #(lf),NAME #(lf) FROM#(lf) RAW.PRODUCTS#(lf)#(lf)) T3#(lf)INNER JOIN#(lf)(#(lf)    SELECT#(lf)    DISTINCT#(lf)    PRESENTMENT_END_DATE#(lf),PRICE_COMMITMENT_PERIOD#(lf),NAME#(lf),CREATED_AT#(lf),DESCRIPTION#(lf),FREE_PERIOD#(lf),WALKOUT#(lf),PRODUCT_CAT_ID#(lf),PROMOTION_ID#(lf),PRESENTMENT_START_DATE#(lf),DEAL_TYPE#(lf),UPDATED_AT#(lf),ID#(lf),STATUS#(lf)    FROM#(lf)    RAW.DEALS#(lf)#(lf)) T2#(lf)ON#(lf)T3.ID   =   T2.PRODUCT_CAT_ID   #(lf)WHERE#(lf)T2.PRESENTMENT_START_DATE >= \'2015-01-01\'#(lf)AND#(lf)T2.STATUS = \'active\'#(lf)#(lf))T4#(lf)ON#(lf)T1.ID   =   T4.PROMOTION_ID#(lf))T5#(lf)INNER JOIN#(lf)RAW.PRICE_MODIFICATIONS T6#(lf)ON#(lf)T5.SEAL_TBL_ID  =   T6.DEAL_ID", null, [EnableFolding=true]) \n in \n Source',
     'let\n Source = Databricks.Catalogs(#"hostname",#"http_path", null),\n edp_prod_Database = Source{[Name=#"catalog",Kind="Database"]}[Data],\n gold_Schema = edp_prod_Database{[Name=#"schema",Kind="Schema"]}[Data],\n pet_view = gold_Schema{[Name="pet_list",Kind="View"]}[Data],\n #"Filtered Rows" = Table.SelectRows(pet_view, each true),\n #"Removed Columns" = Table.RemoveColumns(#"Filtered Rows",{"created_timestmp"})\nin\n #"Removed Columns"',
+    'let\n    Source = Value.NativeQuery(Snowflake.Databases("0DD93C6BD5A6.snowflakecomputing.com","sales_analytics_warehouse_prod",[Role="sales_analytics_member_ad"]){[Name="SL_OPERATIONS"]}[Data], "select SALE_NO AS ""\x1b[4mSaleNo\x1b[0m""#(lf)        ,CODE AS ""Code""#(lf)        ,ENDDATE AS ""end_date""#(lf) from SL_OPERATIONS.SALE.REPORTS#(lf)  where ENDDATE > \'2024-02-03\'", null, [EnableFolding=true]),\n    #"selected Row" = Table.SelectRows(Source)\nin\n    #"selected Row"',
+    'let\n    Source = Value.NativeQuery(DatabricksMultiCloud.Catalogs("foo.com", "/sql/1.0/warehouses/423423ew", [Catalog="sales_db", Database=null, EnableAutomaticProxyDiscovery=null]){[Name="sales_db",Kind="Database"]}[Data], "select * from public.slae_history#(lf)where creation_timestamp  >= getDate(-3)", null, [EnableFolding=true]),\n    #"NewTable" = Table.TransformColumn(Source,{{"creation_timestamp", type date}})\nin\n    #"NewTable"',
+    'let Source = Snowflake.Databases("example.snowflakecomputing.com","WAREHOUSE_NAME",[Role="CUSTOM_ROLE"]), DB_Source = Source{[Name="DATABASE_NAME",Kind="Database"]}[Data], SCHEMA_Source = DB_Source{[Name="SCHEMA_NAME",Kind="Schema"]}[Data], TABLE_Source = SCHEMA_Source{[Name="TABLE_NAME",Kind="View"]}[Data], #"Split Column by Time" = Table.SplitColumn(Table.TransformColumnTypes(TABLE_Source, {{"TIMESTAMP_COLUMN", type text}}, "en-GB"), "TIMESTAMP_COLUMN", Splitter.SplitTextByDelimiter(" ", QuoteStyle.Csv), {"TIMESTAMP_COLUMN.1", "TIMESTAMP_COLUMN.2"}), #"Added Custom" = Table.AddColumn(#"Split Column by Time", "SOB", each ([ENDTIME] - [STARTTIME]) * 60 * 60 * 24) in #"Added Custom"',
+    'let\n    Source = Sql.Database("AUPRDWHDB", "COMMOPSDB", [Query="DROP TABLE IF EXISTS #KKR;#(lf)Select#(lf)*,#(lf)concat((UPPER(REPLACE(SALES_SPECIALIST,\'-\',\'\'))),#(lf)LEFT(CAST(INVOICE_DATE AS DATE),4)+LEFT(RIGHT(CAST(INVOICE_DATE AS DATE),5),2)) AS AGENT_KEY,#(lf)CASE#(lf)    WHEN CLASS = \'Software\' and (NOT(PRODUCT in (\'ADV\', \'Adv\') and left(ACCOUNT_ID,2)=\'10\') #(lf)    or V_ENTERPRISE_INVOICED_REVENUE.TYPE = \'Manual Adjustment\') THEN INVOICE_AMOUNT#(lf)    WHEN V_ENTERPRISE_INVOICED_REVENUE.TYPE IN (\'Recurring\',\'0\') THEN INVOICE_AMOUNT#(lf)    ELSE 0#(lf)END as SOFTWARE_INV#(lf)#(lf)from V_ENTERPRISE_INVOICED_REVENUE", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #"Added Conditional Column" = Table.AddColumn(Source, "Services", each if [CLASS] = "Services" then [INVOICE_AMOUNT] else 0),\n    #"Added Custom" = Table.AddColumn(#"Added Conditional Column", "Advanced New Sites", each if [PRODUCT] = "ADV"\nor [PRODUCT] = "Adv"\nthen [NEW_SITE]\nelse 0)\nin\n    #"Added Custom"',
+    "LOAD_DATA(SOURCE)",
 ]
 
 
+def get_data_platform_tables_with_dummy_table(
+    q: str,
+) -> List[datahub.ingestion.source.powerbi.m_query.data_classes.Lineage]:
+    table: powerbi_data_classes.Table = powerbi_data_classes.Table(
+        columns=[],
+        measures=[],
+        expression=q,
+        name="virtual_order_table",
+        full_name="OrderDataSet.virtual_order_table",
+    )
+
+    reporter = PowerBiDashboardSourceReport()
+
+    ctx, config, platform_instance_resolver = get_default_instances()
+
+    config.enable_advance_lineage_sql_construct = True
+
+    return parser.get_upstream_tables(
+        table,
+        reporter,
+        ctx=ctx,
+        config=config,
+        platform_instance_resolver=platform_instance_resolver,
+    )
+
+
 def get_default_instances(
     override_config: dict = {},
 ) -> Tuple[
@@ -727,7 +765,9 @@ def test_sqlglot_parser():
         }
     )
 
-    lineage: List[resolver.Lineage] = parser.get_upstream_tables(
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = parser.get_upstream_tables(
         table,
         reporter,
         ctx=ctx,
@@ -772,21 +812,15 @@ def test_sqlglot_parser():
 
 
 def test_databricks_multi_cloud():
-    table: powerbi_data_classes.Table = powerbi_data_classes.Table(
-        expression=M_QUERIES[25],
-        name="category",
-        full_name="dev.public.category",
-    )
-    reporter = PowerBiDashboardSourceReport()
+    q = M_QUERIES[25]
 
-    ctx, config, platform_instance_resolver = get_default_instances()
-    data_platform_tables: List[DataPlatformTable] = parser.get_upstream_tables(
-        table,
-        reporter,
-        ctx=ctx,
-        config=config,
-        platform_instance_resolver=platform_instance_resolver,
-    )[0].upstreams
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = get_data_platform_tables_with_dummy_table(q=q)
+
+    assert len(lineage) == 1
+
+    data_platform_tables = lineage[0].upstreams
 
     assert len(data_platform_tables) == 1
 
@@ -797,21 +831,15 @@ def test_databricks_multi_cloud():
 
 
 def test_databricks_catalog_pattern_1():
-    table: powerbi_data_classes.Table = powerbi_data_classes.Table(
-        expression=M_QUERIES[26],
-        name="category",
-        full_name="dev.public.category",
-    )
-    reporter = PowerBiDashboardSourceReport()
+    q = M_QUERIES[26]
 
-    ctx, config, platform_instance_resolver = get_default_instances()
-    data_platform_tables: List[DataPlatformTable] = parser.get_upstream_tables(
-        table,
-        reporter,
-        ctx=ctx,
-        config=config,
-        platform_instance_resolver=platform_instance_resolver,
-    )[0].upstreams
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = get_data_platform_tables_with_dummy_table(q=q)
+
+    assert len(lineage) == 1
+
+    data_platform_tables = lineage[0].upstreams
 
     assert len(data_platform_tables) == 1
 
@@ -876,7 +904,9 @@ def test_sqlglot_parser_2():
         }
     )
 
-    lineage: List[resolver.Lineage] = parser.get_upstream_tables(
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = parser.get_upstream_tables(
         table,
         reporter,
         ctx=ctx,
@@ -929,3 +959,251 @@ def test_databricks_regular_case_with_view():
         data_platform_tables[0].urn
         == "urn:li:dataset:(urn:li:dataPlatform:databricks,cat.public.pet_list,PROD)"
     )
+
+
+@pytest.mark.integration
+def test_snowflake_double_double_quotes():
+    q = M_QUERIES[30]
+
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = get_data_platform_tables_with_dummy_table(q=q)
+
+    assert len(lineage) == 1
+
+    data_platform_tables = lineage[0].upstreams
+
+    assert len(data_platform_tables) == 1
+
+    assert (
+        data_platform_tables[0].urn
+        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,sl_operations.sale.reports,PROD)"
+    )
+
+
+def test_databricks_multicloud():
+    q = M_QUERIES[31]
+
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = get_data_platform_tables_with_dummy_table(q=q)
+
+    assert len(lineage) == 1
+
+    data_platform_tables = lineage[0].upstreams
+
+    assert len(data_platform_tables) == 1
+
+    assert (
+        data_platform_tables[0].urn
+        == "urn:li:dataset:(urn:li:dataPlatform:databricks,sales_db.public.slae_history,PROD)"
+    )
+
+
+def test_snowflake_multi_function_call():
+    q = M_QUERIES[32]
+
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = get_data_platform_tables_with_dummy_table(q=q)
+
+    assert len(lineage) == 1
+
+    data_platform_tables = lineage[0].upstreams
+
+    assert len(data_platform_tables) == 1
+
+    assert (
+        data_platform_tables[0].urn
+        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,database_name.schema_name.table_name,PROD)"
+    )
+
+
+def test_mssql_drop_with_select():
+    q = M_QUERIES[33]
+
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = get_data_platform_tables_with_dummy_table(q=q)
+
+    assert len(lineage) == 1
+
+    data_platform_tables = lineage[0].upstreams
+
+    assert len(data_platform_tables) == 1
+
+    assert (
+        data_platform_tables[0].urn
+        == "urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_enterprise_invoiced_revenue,PROD)"
+    )
+
+
+def test_unsupported_data_platform():
+    q = M_QUERIES[34]
+    table: powerbi_data_classes.Table = powerbi_data_classes.Table(
+        columns=[],
+        measures=[],
+        expression=q,
+        name="virtual_order_table",
+        full_name="OrderDataSet.virtual_order_table",
+    )
+
+    reporter = PowerBiDashboardSourceReport()
+
+    ctx, config, platform_instance_resolver = get_default_instances()
+
+    config.enable_advance_lineage_sql_construct = True
+
+    assert (
+        parser.get_upstream_tables(
+            table,
+            reporter,
+            ctx=ctx,
+            config=config,
+            platform_instance_resolver=platform_instance_resolver,
+        )
+        == []
+    )
+
+    info_entries: dict = reporter._structured_logs._entries.get(
+        StructuredLogLevel.INFO, {}
+    )  # type :ignore
+
+    is_entry_present: bool = False
+    for key, entry in info_entries.items():
+        if entry.title == "Non-Data Platform Expression":
+            is_entry_present = True
+            break
+
+    assert (
+        is_entry_present
+    ), 'Info message "Non-Data Platform Expression" should be present in reporter'
+
+
+def test_empty_string_in_m_query():
+    # TRIM(TRIM(TRIM(AGENT_NAME, '\"\"'), '+'), '\\'') is in Query
+    q = "let\n  Source = Value.NativeQuery(Snowflake.Databases(\"bu10758.ap-unknown-2.fakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"select #(lf)UPPER(REPLACE(AGENT_NAME,'-','')) AS CLIENT_DIRECTOR,#(lf)TRIM(TRIM(TRIM(AGENT_NAME, '\"\"'), '+'), '\\'') AS TRIM_AGENT_NAME,#(lf)TIER,#(lf)UPPER(MANAGER),#(lf)TEAM_TYPE,#(lf)DATE_TARGET,#(lf)MONTHID,#(lf)TARGET_TEAM,#(lf)SELLER_EMAIL,#(lf)concat((UPPER(REPLACE(AGENT_NAME,'-',''))), MONTHID) as AGENT_KEY,#(lf)UNIT_TARGET AS SME_Quota,#(lf)AMV_TARGET AS Revenue_Quota,#(lf)SERVICE_QUOTA,#(lf)BL_TARGET,#(lf)SOFTWARE_QUOTA as Software_Quota#(lf)#(lf)from OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_SME_UNIT_TARGETS inner join OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_SME_UNIT #(lf)#(lf)where YEAR_TARGET >= 2022#(lf)and TEAM_TYPE = 'Accounting'#(lf)and TARGET_TEAM = 'Enterprise'#(lf)AND TIER = 'Client Director'\", null, [EnableFolding=true])\nin\n    Source"
+
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = get_data_platform_tables_with_dummy_table(q=q)
+
+    assert len(lineage) == 1
+
+    data_platform_tables = lineage[0].upstreams
+
+    assert len(data_platform_tables) == 2
+
+    assert (
+        data_platform_tables[0].urn
+        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,operations_analytics.transformed_prod.v_sme_unit,PROD)"
+    )
+    assert (
+        data_platform_tables[1].urn
+        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,operations_analytics.transformed_prod.v_sme_unit_targets,PROD)"
+    )
+
+
+def test_double_quotes_in_alias():
+    # SELECT CAST(sales_date AS DATE) AS \"\"Date\"\" in query
+    q = 'let \n Source = Sql.Database("abc.com", "DB", [Query="SELECT CAST(sales_date AS DATE) AS ""Date"",#(lf) SUM(cshintrpret) / 60.0      AS ""Total Order All Items"",#(lf)#(tab)#(tab)#(tab)  SUM(cshintrpret) / 60.0 - LAG(SUM(cshintrpret) / 60.0, 1) OVER (ORDER BY CAST(sales_date AS DATE)) AS ""Total minute difference"",#(lf)#(tab)#(tab)#(tab)  SUM(sale_price)  / 60.0 - LAG(SUM(sale_price)  / 60.0, 1) OVER (ORDER BY CAST(sales_date AS DATE)) AS ""Normal minute difference""#(lf)        FROM   [DB].[dbo].[sales_t]#(lf)        WHERE  sales_date >= GETDATE() - 365#(lf)        GROUP  BY CAST(sales_date AS DATE),#(lf)#(tab)#(tab)CAST(sales_date AS TIME);"]) \n in \n Source'
+
+    lineage: List[
+        datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
+    ] = get_data_platform_tables_with_dummy_table(q=q)
+
+    assert len(lineage) == 1
+
+    data_platform_tables = lineage[0].upstreams
+
+    assert len(data_platform_tables) == 1
+
+    assert (
+        data_platform_tables[0].urn
+        == "urn:li:dataset:(urn:li:dataPlatform:mssql,db.dbo.sales_t,PROD)"
+    )
+
+
+@patch("datahub.ingestion.source.powerbi.m_query.parser.get_lark_parser")
+def test_m_query_timeout(mock_get_lark_parser):
+    q = 'let\n    Source = Value.NativeQuery(Snowflake.Databases("0DD93C6BD5A6.snowflakecomputing.com","sales_analytics_warehouse_prod",[Role="sales_analytics_member_ad"]){[Name="SL_OPERATIONS"]}[Data], "select SALE_NO AS ""\x1b[4mSaleNo\x1b[0m""#(lf)        ,CODE AS ""Code""#(lf)        ,ENDDATE AS ""end_date""#(lf) from SL_OPERATIONS.SALE.REPORTS#(lf)  where ENDDATE > \'2024-02-03\'", null, [EnableFolding=true]),\n    #"selected Row" = Table.SelectRows(Source)\nin\n    #"selected Row"'
+
+    table: powerbi_data_classes.Table = powerbi_data_classes.Table(
+        columns=[],
+        measures=[],
+        expression=q,
+        name="virtual_order_table",
+        full_name="OrderDataSet.virtual_order_table",
+    )
+
+    reporter = PowerBiDashboardSourceReport()
+
+    ctx, config, platform_instance_resolver = get_default_instances()
+
+    config.enable_advance_lineage_sql_construct = True
+
+    config.m_query_parse_timeout = 1
+
+    mock_lark_instance = MagicMock()
+
+    mock_get_lark_parser.return_value = mock_lark_instance
+    # sleep for 5 seconds to trigger timeout
+    mock_lark_instance.parse.side_effect = lambda expression: time.sleep(5)
+
+    parser.get_upstream_tables(
+        table,
+        reporter,
+        ctx=ctx,
+        config=config,
+        platform_instance_resolver=platform_instance_resolver,
+    )
+
+    warn_entries: dict = reporter._structured_logs._entries.get(
+        StructuredLogLevel.WARN, {}
+    )  # type :ignore
+
+    is_entry_present: bool = False
+    for key, entry in warn_entries.items():
+        if entry.title == "M-Query Parsing Timeout":
+            is_entry_present = True
+            break
+
+    assert (
+        is_entry_present
+    ), 'Warning message "M-Query Parsing Timeout" should be present in reporter'
+
+
+def test_comments_in_m_query():
+    q: str = 'let\n    Source = Snowflake.Databases("xaa48144.snowflakecomputing.com", "COMPUTE_WH", [Role="ACCOUNTADMIN"]),\n    SNOWFLAKE_SAMPLE_DATA_Database = Source{[Name="SNOWFLAKE_SAMPLE_DATA", Kind="Database"]}[Data],\n    TPCDS_SF100TCL_Schema = SNOWFLAKE_SAMPLE_DATA_Database{[Name="TPCDS_SF100TCL", Kind="Schema"]}[Data],\n    ITEM_Table = TPCDS_SF100TCL_Schema{[Name="ITEM", Kind="Table"]}[Data],\n    \n    // Group by I_BRAND and calculate the count\n    BrandCountsTable = Table.Group(ITEM_Table, {"I_BRAND"}, {{"BrandCount", each Table.RowCount(_), Int64.Type}})\nin\n    BrandCountsTable'
+
+    table: powerbi_data_classes.Table = powerbi_data_classes.Table(
+        columns=[],
+        measures=[],
+        expression=q,
+        name="pet_price_index",
+        full_name="datalake.sandbox_pet.pet_price_index",
+    )
+
+    reporter = PowerBiDashboardSourceReport()
+
+    ctx, config, platform_instance_resolver = get_default_instances()
+
+    data_platform_tables: List[DataPlatformTable] = parser.get_upstream_tables(
+        table,
+        reporter,
+        ctx=ctx,
+        config=config,
+        platform_instance_resolver=platform_instance_resolver,
+        parameters={
+            "hostname": "xyz.databricks.com",
+            "http_path": "/sql/1.0/warehouses/abc",
+            "catalog": "cat",
+            "schema": "public",
+        },
+    )[0].upstreams
+
+    assert len(data_platform_tables) == 1
+    assert (
+        data_platform_tables[0].urn
+        == "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpcds_sf100tcl.item,PROD)"
+    )
diff --git a/metadata-ingestion/tests/integration/powerbi/test_native_sql_parser.py b/metadata-ingestion/tests/integration/powerbi/test_native_sql_parser.py
index 53e184515c1d8..887f7fe4d6f44 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_native_sql_parser.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_native_sql_parser.py
@@ -19,3 +19,13 @@ def test_simple_from():
 
     assert len(tables) == 1
     assert tables[0] == "OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4"
+
+
+def test_drop_statement():
+    expected: str = "SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4"
+
+    query: str = "DROP TABLE IF EXISTS #table1; DROP TABLE IF EXISTS #table1,#table2; DROP TABLE IF EXISTS table1; DROP TABLE IF EXISTS table1, #table2;SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4"
+
+    actual: str = native_sql_parser.remove_drop_statement(query)
+
+    assert actual == expected
diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
index 23b23ecada0d4..edde11ff87d29 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
@@ -1,14 +1,17 @@
 import datetime
+import json
 import logging
 import re
 import sys
-from typing import Any, Dict, List, Optional, cast
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union, cast
 from unittest import mock
 from unittest.mock import MagicMock
 
 import pytest
 from freezegun import freeze_time
 
+from datahub.ingestion.api.source import StructuredLogLevel
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.ingestion.source.powerbi.config import (
     Constant,
@@ -19,6 +22,7 @@
 from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import (
     Page,
     Report,
+    ReportType,
     Workspace,
 )
 from tests.test_helpers import mce_helpers, test_connection_helpers
@@ -70,549 +74,51 @@ def scan_init_response(request, context):
         "64ED5CAD-7C10-4684-8180-826122881108||64ED5CAD-7C22-4684-8180-826122881108": {
             "id": "a674efd1-603c-4129-8d82-03cf2be05aff"
         },
+        "A8D655A6-F521-477E-8C22-255018583BF4": {
+            "id": "62DAF926-0B18-4FF1-982C-2A3EB6B8F0E4"
+        },
+        "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492": {
+            "id": "81B02907-E2A3-45C3-B505-3781839C8CAA",
+        },
+        "8F756DE6-26AD-45FF-A201-44276FF1F561": {
+            "id": "6147FCEB-7531-4449-8FB6-1F7A5431BF2D",
+        },
     }
 
     return w_id_vs_response[workspace_id]
 
 
-def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -> None:
-    override_data = override_data or {}
+def read_mock_data(path: Union[Path, str]) -> dict:
+    with open(path) as p:
+        return json.load(p)
+
+
+def register_mock_api(
+    pytestconfig: pytest.Config, request_mock: Any, override_data: Optional[dict] = None
+) -> None:
+    default_mock_data_path = (
+        pytestconfig.rootpath
+        / "tests/integration/powerbi/mock_data/default_mock_response.json"
+    )
+
     api_vs_response = {
-        "https://api.powerbi.com/v1.0/myorg/groups": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "@odata.count": 3,
-                "value": [
-                    {
-                        "id": "64ED5CAD-7C10-4684-8180-826122881108",
-                        "isReadOnly": True,
-                        "name": "demo-workspace",
-                        "type": "Workspace",
-                    },
-                    {
-                        "id": "64ED5CAD-7C22-4684-8180-826122881108",
-                        "isReadOnly": True,
-                        "name": "second-demo-workspace",
-                        "type": "Workspace",
-                    },
-                    {
-                        "id": "64ED5CAD-7322-4684-8180-826122881108",
-                        "isReadOnly": True,
-                        "name": "Workspace 2",
-                        "type": "Workspace",
-                    },
-                ],
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE",
-                        "isReadOnly": True,
-                        "displayName": "test_dashboard",
-                        "description": "Description of test dashboard",
-                        "embedUrl": "https://localhost/dashboards/embed/1",
-                        "webUrl": "https://localhost/dashboards/web/1",
-                    }
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE",
-                        "isReadOnly": True,
-                        "displayName": "test_dashboard2",
-                        "embedUrl": "https://localhost/dashboards/embed/1",
-                        "webUrl": "https://localhost/dashboards/web/1",
-                    }
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/admin/reports/5b218778-e7a5-4d73-8187-f10824047715/users": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "identifier": "User1@foo.com",
-                        "displayName": "user1",
-                        "emailAddress": "User1@foo.com",
-                        "datasetUserAccessRight": "ReadWrite",
-                        "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
-                        "principalType": "User",
-                    },
-                    {
-                        "identifier": "User2@foo.com",
-                        "displayName": "user2",
-                        "emailAddress": "User2@foo.com",
-                        "datasetUserAccessRight": "ReadWrite",
-                        "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
-                        "principalType": "User",
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/users": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "identifier": "User1@foo.com",
-                        "displayName": "user1",
-                        "emailAddress": "User1@foo.com",
-                        "datasetUserAccessRight": "ReadWrite",
-                        "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
-                        "principalType": "User",
-                    },
-                    {
-                        "identifier": "User2@foo.com",
-                        "displayName": "user2",
-                        "emailAddress": "User2@foo.com",
-                        "datasetUserAccessRight": "ReadWrite",
-                        "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
-                        "principalType": "User",
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/users": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "identifier": "User3@foo.com",
-                        "displayName": "user3",
-                        "emailAddress": "User3@foo.com",
-                        "datasetUserAccessRight": "ReadWrite",
-                        "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
-                        "principalType": "User",
-                    },
-                    {
-                        "identifier": "User4@foo.com",
-                        "displayName": "user4",
-                        "emailAddress": "User4@foo.com",
-                        "datasetUserAccessRight": "ReadWrite",
-                        "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
-                        "principalType": "User",
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "id": "B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0",
-                        "title": "test_tile",
-                        "embedUrl": "https://localhost/tiles/embed/1",
-                        "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                    },
-                    {
-                        "id": "23212598-23b5-4980-87cc-5fc0ecd84385",
-                        "title": "yearly_sales",
-                        "embedUrl": "https://localhost/tiles/embed/2",
-                        "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/tiles": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {"value": []},
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "id": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "name": "library-dataset",
-                "description": "Library dataset description",
-                "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445",
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "id": "05169CD2-E713-41E6-96AA-1D8066D95445",
-                "name": "library-dataset",
-                "description": "Library dataset description",
-                "webUrl": "http://localhost/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445",
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                "name": "hr_pbi_test",
-                "description": "hr pbi test description",
-                "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed",
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/datasources": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                        "datasourceType": "PostgreSql",
-                        "connectionDetails": {
-                            "database": "library_db",
-                            "server": "foo",
-                        },
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445/datasources": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                        "datasourceType": "PostgreSql",
-                        "connectionDetails": {
-                            "database": "library_db",
-                            "server": "foo",
-                        },
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4674efd1-603c-4129-8d82-03cf2be05aff": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "status": "SUCCEEDED",
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/a674efd1-603c-4129-8d82-03cf2be05aff": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "status": "SUCCEEDED",
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "workspaces": [
-                    {
-                        "id": "64ED5CAD-7C10-4684-8180-826122881108",
-                        "name": "demo-workspace",
-                        "state": "Active",
-                        "datasets": [
-                            {
-                                "id": "05169CD2-E713-41E6-9600-1D8066D95445",
-                                "endorsementDetails": {"endorsement": "Promoted"},
-                                "name": "test_sf_pbi_test",
-                                "tables": [
-                                    {
-                                        "name": "public issue_history",
-                                        "source": [
-                                            {
-                                                "expression": "dummy",
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                    {
-                                        "name": "SNOWFLAKE_TESTTABLE",
-                                        "source": [
-                                            {
-                                                "expression": 'let\n    Source = Snowflake.Databases("hp123rt5.ap-southeast-2.fakecomputing.com","PBI_TEST_WAREHOUSE_PROD",[Role="PBI_TEST_MEMBER"]),\n    PBI_TEST_Database = Source{[Name="PBI_TEST",Kind="Database"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name="TEST",Kind="Schema"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name="TESTTABLE",Kind="Table"]}[Data]\nin\n    TESTTABLE_Table',
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                    {
-                                        "name": "snowflake native-query",
-                                        "source": [
-                                            {
-                                                "expression": 'let\n    Source = Value.NativeQuery(Snowflake.Databases("bu20658.ap-southeast-2.snowflakecomputing.com","operations_analytics_warehouse_prod",[Role="OPERATIONS_ANALYTICS_MEMBER"]){[Name="OPERATIONS_ANALYTICS"]}[Data], "SELECT#(lf)concat((UPPER(REPLACE(SELLER,\'-\',\'\'))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,\'-\',\'\'))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4", null, [EnableFolding=true]),\n    #"Added Conditional Column" = Table.AddColumn(Source, "SME Units ENT", each if [DEAL_TYPE] = "SME Unit" then [UNIT] else 0),\n    #"Added Conditional Column1" = Table.AddColumn(#"Added Conditional Column", "Banklink Units", each if [DEAL_TYPE] = "Banklink" then [UNIT] else 0),\n    #"Removed Columns" = Table.RemoveColumns(#"Added Conditional Column1",{"Banklink Units"}),\n    #"Added Custom" = Table.AddColumn(#"Removed Columns", "Banklink Units", each if [DEAL_TYPE] = "Banklink" and [SALES_TYPE] = "3 - Upsell"\nthen [UNIT]\n\nelse if [SALES_TYPE] = "Adjusted BL Migration"\nthen [UNIT]\n\nelse 0),\n    #"Added Custom1" = Table.AddColumn(#"Added Custom", "SME Units in $ (*$361)", each if [DEAL_TYPE] = "SME Unit" \nand [SALES_TYPE] <> "4 - Renewal"\n    then [UNIT] * 361\nelse 0),\n    #"Added Custom2" = Table.AddColumn(#"Added Custom1", "Banklink in $ (*$148)", each [Banklink Units] * 148)\nin\n    #"Added Custom2"',
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                    {
-                                        "name": "big-query-with-parameter",
-                                        "source": [
-                                            {
-                                                "expression": 'let\n Source = GoogleBigQuery.Database([BillingProject = #"Parameter - Source"]),\n#"gcp-project" = Source{[Name=#"Parameter - Source"]}[Data],\nuniversal_Schema = #"gcp-project"{[Name="universal",Kind="Schema"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name="D_WH_DATE",Kind="Table"]}[Data],\n#"Filtered Rows" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#"Filtered Rows1" = Table.SelectRows(#"Filtered Rows", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#"Filtered Rows1"',
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                    {
-                                        "name": "snowflake native-query-with-join",
-                                        "source": [
-                                            {
-                                                "expression": 'let\n    Source = Value.NativeQuery(Snowflake.Databases("xaa48144.snowflakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]){[Name="GSL_TEST_DB"]}[Data], "select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, \'mo\')", null, [EnableFolding=true])\nin\n    Source',
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                    {
-                                        "name": "job-history",
-                                        "source": [
-                                            {
-                                                "expression": 'let\n    Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                    {
-                                        "name": "postgres_test_table",
-                                        "source": [
-                                            {
-                                                "expression": 'let\n    Source = PostgreSQL.Database("localhost"  ,   "mics"      ),\n  public_order_date =    Source{[Schema="public",Item="order_date"]}[Data] \n in \n public_order_date',
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                ],
-                            },
-                            {
-                                "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
-                                "name": "hr_pbi_test",
-                                "tables": [
-                                    {
-                                        "name": "dbo_book_issue",
-                                        "source": [
-                                            {
-                                                "expression": 'let\n    Source = Sql.Database("localhost", "library"),\n dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]\n in dbo_book_issue',
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                    {
-                                        "name": "ms_sql_native_table",
-                                        "source": [
-                                            {
-                                                "expression": 'let\n    Source = Sql.Database("AUPRDWHDB", "COMMOPSDB", [Query="select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,\'-\',\'\'))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,\'-\',\'\'))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION", CommandTimeout=#duration(0, 1, 30, 0)]),\n    #"Changed Type" = Table.TransformColumnTypes(Source,{{"mth_date", type date}}),\n    #"Added Custom" = Table.AddColumn(#"Changed Type", "Month", each Date.Month([mth_date])),\n    #"Added Custom1" = Table.AddColumn(#"Added Custom", "TPV Opening", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n    #"Added Custom1"',
-                                            }
-                                        ],
-                                        "datasourceUsages": [
-                                            {
-                                                "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
-                                            }
-                                        ],
-                                    },
-                                ],
-                            },
-                            {
-                                "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
-                                "tables": [
-                                    {
-                                        "name": "employee_ctc",
-                                        "source": [
-                                            {
-                                                "expression": "dummy",
-                                            }
-                                        ],
-                                    }
-                                ],
-                            },
-                        ],
-                        "dashboards": [
-                            {
-                                "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE",
-                                "isReadOnly": True,
-                            }
-                        ],
-                        "reports": [
-                            {
-                                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                                "id": "5b218778-e7a5-4d73-8187-f10824047715",
-                                "name": "SalesMarketing",
-                                "description": "Acryl sales marketing report",
-                            }
-                        ],
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/a674efd1-603c-4129-8d82-03cf2be05aff": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "workspaces": [
-                    {
-                        "id": "64ED5CAD-7C22-4684-8180-826122881108",
-                        "name": "second-demo-workspace",
-                        "state": "Active",
-                        "datasets": [
-                            {
-                                "id": "05169CD2-E713-41E6-96AA-1D8066D95445",
-                                "tables": [
-                                    {
-                                        "name": "public articles",
-                                        "source": [
-                                            {
-                                                "expression": "dummy",
-                                            }
-                                        ],
-                                    }
-                                ],
-                            }
-                        ],
-                        "dashboards": [
-                            {
-                                "id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE",
-                                "isReadOnly": True,
-                            }
-                        ],
-                        "reports": [
-                            {
-                                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                                "id": "5b218778-e7a5-4d73-8187-f10824047715",
-                                "name": "SalesMarketing",
-                                "description": "Acryl sales marketing report",
-                            }
-                        ],
-                    },
-                ]
-            },
-        },
         "https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo": {
             "method": "POST",
             "status_code": 200,
             "json": scan_init_response,
         },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                        "id": "5b218778-e7a5-4d73-8187-f10824047715",
-                        "name": "SalesMarketing",
-                        "description": "Acryl sales marketing report",
-                        "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715",
-                        "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48",
-                    }
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
-                "id": "5b218778-e7a5-4d73-8187-f10824047715",
-                "name": "SalesMarketing",
-                "description": "Acryl sales marketing report",
-                "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715",
-                "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48",
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "displayName": "Regional Sales Analysis",
-                        "name": "ReportSection",
-                        "order": "0",
-                    },
-                    {
-                        "displayName": "Geographic Analysis",
-                        "name": "ReportSection1",
-                        "order": "1",
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/parameters": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "value": [
-                    {
-                        "name": "Parameter - Source",
-                        "type": "Text",
-                        "isRequired": True,
-                        "currentValue": "my-test-project",
-                    },
-                    {
-                        "name": "My bq project",
-                        "type": "Text",
-                        "isRequired": True,
-                        "currentValue": "gcp_billing",
-                    },
-                ]
-            },
-        },
-        "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff": {
-            "method": "GET",
-            "status_code": 200,
-            "json": {
-                "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
-                "name": "employee-dataset",
-                "description": "Employee Management",
-                "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
-            },
-        },
     }
 
-    api_vs_response.update(override_data)
+    api_vs_response.update(read_mock_data(default_mock_data_path))
+
+    api_vs_response.update(override_data or {})
 
     for url in api_vs_response.keys():
         request_mock.register_uri(
             api_vs_response[url]["method"],
             url,
-            json=api_vs_response[url]["json"],
+            json=api_vs_response[url].get("json"),
+            text=api_vs_response[url].get("text"),
             status_code=api_vs_response[url]["status_code"],
         )
 
@@ -652,7 +158,7 @@ def test_powerbi_ingest(
 
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -683,6 +189,67 @@ def test_powerbi_ingest(
     )
 
 
+@freeze_time(FROZEN_TIME)
+@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
+@pytest.mark.integration
+def test_powerbi_workspace_type_filter(
+    mock_msal: MagicMock,
+    pytestconfig: pytest.Config,
+    tmp_path: str,
+    mock_time: datetime.datetime,
+    requests_mock: Any,
+) -> None:
+    enable_logging()
+
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
+
+    register_mock_api(
+        request_mock=requests_mock,
+        pytestconfig=pytestconfig,
+        override_data=read_mock_data(
+            pytestconfig.rootpath
+            / "tests/integration/powerbi/mock_data/workspace_type_filter.json"
+        ),
+    )
+
+    default_config: dict = default_source_config()
+
+    del default_config["workspace_id"]
+    del default_config["workspace_id_pattern"]
+
+    pipeline = Pipeline.create(
+        {
+            "run_id": "powerbi-test",
+            "source": {
+                "type": "powerbi",
+                "config": {
+                    **default_config,
+                    "extract_workspaces_to_containers": True,
+                    "workspace_type_filter": [
+                        "PersonalGroup",
+                    ],
+                },
+            },
+            "sink": {
+                "type": "file",
+                "config": {
+                    "filename": f"{tmp_path}/powerbi_mces.json",
+                },
+            },
+        }
+    )
+
+    pipeline.run()
+    pipeline.raise_from_status()
+    golden_file = "golden_test_personal_ingest.json"
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=f"{tmp_path}/powerbi_mces.json",
+        golden_path=f"{test_resources_dir}/{golden_file}",
+    )
+
+
 @freeze_time(FROZEN_TIME)
 @mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
 @pytest.mark.integration
@@ -697,7 +264,7 @@ def test_powerbi_ingest_patch_disabled(
 
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -764,7 +331,7 @@ def test_powerbi_platform_instance_ingest(
 
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     output_path: str = f"{tmp_path}/powerbi_platform_instance_mces.json"
 
@@ -810,7 +377,7 @@ def test_powerbi_ingest_urn_lower_case(
 ) -> None:
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -857,7 +424,7 @@ def test_override_ownership(
 ) -> None:
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -899,10 +466,9 @@ def test_scan_all_workspaces(
     mock_time: datetime.datetime,
     requests_mock: Any,
 ) -> None:
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -949,12 +515,11 @@ def test_extract_reports(
     mock_time: datetime.datetime,
     requests_mock: Any,
 ) -> None:
-
     enable_logging()
 
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -1000,7 +565,7 @@ def test_extract_lineage(
 
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -1052,7 +617,7 @@ def test_extract_endorsements(
 ) -> None:
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -1100,6 +665,7 @@ def test_admin_access_is_not_allowed(
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
     register_mock_api(
+        pytestconfig=pytestconfig,
         request_mock=requests_mock,
         override_data={
             "https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo": {
@@ -1161,7 +727,7 @@ def test_workspace_container(
 
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -1208,7 +774,7 @@ def test_access_token_expiry_with_long_expiry(
 ) -> None:
     enable_logging()
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -1250,7 +816,7 @@ def test_access_token_expiry_with_short_expiry(
 ) -> None:
     enable_logging()
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     pipeline = Pipeline.create(
         {
@@ -1305,7 +871,7 @@ def test_dataset_type_mapping_should_set_to_all(
     """
     Here we don't need to run the pipeline. We need to verify dataset_type_mapping is set to default dataplatform
     """
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     new_config: dict = {**default_source_config()}
 
@@ -1342,7 +908,7 @@ def test_dataset_type_mapping_error(
     Here we don't need to run the pipeline. We need to verify if both dataset_type_mapping and server_to_platform_instance
     are set then value error should get raised
     """
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     with pytest.raises(Exception, match=r"dataset_type_mapping is deprecated"):
         Pipeline.create(
@@ -1397,7 +963,7 @@ def test_server_to_platform_map(
         "localhost:1521": {"platform_instance": "oracle-sales-instance", "env": "PROD"},
     }
 
-    register_mock_api(request_mock=requests_mock)
+    register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock)
 
     output_path: str = f"{tmp_path}/powerbi_server_to_platform_instance_mces.json"
 
@@ -1439,6 +1005,7 @@ def validate_pipeline(pipeline: Pipeline) -> None:
     mock_workspace: Workspace = Workspace(
         id="64ED5CAD-7C10-4684-8180-826122881108",
         name="demo-workspace",
+        type="Workspace",
         datasets={},
         dashboards=[],
         reports=[],
@@ -1446,6 +1013,7 @@ def validate_pipeline(pipeline: Pipeline) -> None:
         dashboard_endorsements={},
         scan_result={},
         independent_datasets=[],
+        app=None,
     )
     # Fetch actual reports
     reports: List[Report] = cast(
@@ -1485,6 +1053,7 @@ def validate_pipeline(pipeline: Pipeline) -> None:
         Report(
             id=report[Constant.ID],
             name=report[Constant.NAME],
+            type=ReportType.PowerBIReport,
             webUrl="",
             embedUrl="",
             description=report[Constant.DESCRIPTION],
@@ -1501,6 +1070,7 @@ def validate_pipeline(pipeline: Pipeline) -> None:
             ],
             users=[],
             tags=[],
+            dataset_id=report[Constant.DATASET_ID],
             dataset=mock_workspace.datasets.get(report[Constant.DATASET_ID]),
         )
         for report in mock_reports
@@ -1528,6 +1098,7 @@ def test_reports_with_failed_page_request(
     Test that all reports are fetched even if a single page request fails
     """
     register_mock_api(
+        pytestconfig=pytestconfig,
         request_mock=requests_mock,
         override_data={
             "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": {
@@ -1538,6 +1109,7 @@ def test_reports_with_failed_page_request(
                         {
                             "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
                             "id": "5b218778-e7a5-4d73-8187-f10824047715",
+                            "reportType": "PowerBIReport",
                             "name": "SalesMarketing",
                             "description": "Acryl sales marketing report",
                             "webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715",
@@ -1546,6 +1118,7 @@ def test_reports_with_failed_page_request(
                         {
                             "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
                             "id": "e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
+                            "reportType": "PaginatedReport",
                             "name": "Product",
                             "description": "Acryl product report",
                             "webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
@@ -1561,6 +1134,7 @@ def test_reports_with_failed_page_request(
                     "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
                     "id": "5b218778-e7a5-4d73-8187-f10824047715",
                     "name": "SalesMarketing",
+                    "reportType": "PowerBIReport",
                     "description": "Acryl sales marketing report",
                     "webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715",
                     "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=64ED5CAD-7C10-4684-8180-826122881108",
@@ -1572,6 +1146,7 @@ def test_reports_with_failed_page_request(
                 "json": {
                     "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
                     "id": "e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
+                    "reportType": "PowerBIReport",
                     "name": "Product",
                     "description": "Acryl product report",
                     "webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
@@ -1641,17 +1216,16 @@ def test_independent_datasets_extraction(
     mock_time: datetime.datetime,
     requests_mock: Any,
 ) -> None:
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
     register_mock_api(
+        pytestconfig=pytestconfig,
         request_mock=requests_mock,
         override_data={
-            "https://api.powerbi.com/v1.0/myorg/groups": {
+            "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": {
                 "method": "GET",
                 "status_code": 200,
                 "json": {
-                    "@odata.count": 3,
                     "value": [
                         {
                             "id": "64ED5CAD-7C10-4684-8180-826122881108",
@@ -1662,6 +1236,13 @@ def test_independent_datasets_extraction(
                     ],
                 },
             },
+            "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": {
+                "method": "GET",
+                "status_code": 200,
+                "json": {
+                    "value": [],
+                },
+            },
             "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": {
                 "method": "GET",
                 "status_code": 200,
@@ -1670,6 +1251,7 @@ def test_independent_datasets_extraction(
                         {
                             "id": "64ED5CAD-7C10-4684-8180-826122881108",
                             "name": "demo-workspace",
+                            "type": "Workspace",
                             "state": "Active",
                             "datasets": [
                                 {
@@ -1737,10 +1319,10 @@ def test_cll_extraction(
     mock_time: datetime.datetime,
     requests_mock: Any,
 ) -> None:
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
 
     register_mock_api(
+        pytestconfig=pytestconfig,
         request_mock=requests_mock,
     )
 
@@ -1793,8 +1375,8 @@ def test_cll_extraction_flags(
     mock_time: datetime.datetime,
     requests_mock: Any,
 ) -> None:
-
     register_mock_api(
+        pytestconfig=pytestconfig,
         request_mock=requests_mock,
     )
 
@@ -1804,7 +1386,6 @@ def test_cll_extraction_flags(
     )
 
     with pytest.raises(Exception, match=pattern):
-
         Pipeline.create(
             {
                 "run_id": "powerbi-test",
@@ -1823,3 +1404,203 @@ def test_cll_extraction_flags(
                 },
             }
         )
+
+
+@freeze_time(FROZEN_TIME)
+@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
+@pytest.mark.integration
+def test_powerbi_cross_workspace_reference_info_message(
+    mock_msal: MagicMock,
+    pytestconfig: pytest.Config,
+    tmp_path: str,
+    mock_time: datetime.datetime,
+    requests_mock: Any,
+) -> None:
+    enable_logging()
+
+    register_mock_api(
+        pytestconfig=pytestconfig,
+        request_mock=requests_mock,
+        override_data=read_mock_data(
+            path=pytestconfig.rootpath
+            / "tests/integration/powerbi/mock_data/cross_workspace_mock_response.json"
+        ),
+    )
+
+    config = default_source_config()
+
+    del config["workspace_id"]
+
+    config["workspace_id_pattern"] = {
+        "allow": [
+            "A8D655A6-F521-477E-8C22-255018583BF4",
+            "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492",
+        ]
+    }
+
+    config["include_workspace_name_in_dataset_urn"] = True
+
+    pipeline = Pipeline.create(
+        {
+            "run_id": "powerbi-test",
+            "source": {
+                "type": "powerbi",
+                "config": {
+                    **config,
+                },
+            },
+            "sink": {
+                "type": "file",
+                "config": {
+                    "filename": f"{tmp_path}/powerbi_mces.json",
+                },
+            },
+        }
+    )
+
+    pipeline.run()
+    pipeline.raise_from_status()
+
+    assert isinstance(pipeline.source, PowerBiDashboardSource)  # to silent the lint
+
+    info_entries: dict = pipeline.source.reporter._structured_logs._entries.get(
+        StructuredLogLevel.INFO, {}
+    )  # type :ignore
+
+    is_entry_present: bool = False
+    # Printing INFO entries
+    for key, entry in info_entries.items():
+        if entry.title == "Missing Lineage For Tile":
+            is_entry_present = True
+            break
+
+    assert (
+        is_entry_present
+    ), 'Info message "Missing Lineage For Tile" should be present in reporter'
+
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
+
+    golden_file = "golden_test_cross_workspace_dataset.json"
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=f"{tmp_path}/powerbi_mces.json",
+        golden_path=f"{test_resources_dir}/{golden_file}",
+    )
+
+
+def common_app_ingest(
+    pytestconfig: pytest.Config,
+    requests_mock: Any,
+    output_mcp_path: str,
+    override_config: dict = {},
+) -> Pipeline:
+    enable_logging()
+
+    register_mock_api(
+        pytestconfig=pytestconfig,
+        request_mock=requests_mock,
+        override_data=read_mock_data(
+            path=pytestconfig.rootpath
+            / "tests/integration/powerbi/mock_data/workspace_with_app_mock_response.json"
+        ),
+    )
+
+    config = default_source_config()
+
+    del config["workspace_id"]
+
+    config["workspace_id_pattern"] = {
+        "allow": [
+            "8F756DE6-26AD-45FF-A201-44276FF1F561",
+        ]
+    }
+
+    config.update(override_config)
+
+    pipeline = Pipeline.create(
+        {
+            "run_id": "powerbi-test",
+            "source": {
+                "type": "powerbi",
+                "config": {
+                    **config,
+                },
+            },
+            "sink": {
+                "type": "file",
+                "config": {
+                    "filename": output_mcp_path,
+                },
+            },
+        }
+    )
+
+    pipeline.run()
+    pipeline.raise_from_status()
+
+    return pipeline
+
+
+@freeze_time(FROZEN_TIME)
+@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
+@pytest.mark.integration
+def test_powerbi_app_ingest(
+    mock_msal: MagicMock,
+    pytestconfig: pytest.Config,
+    tmp_path: str,
+    mock_time: datetime.datetime,
+    requests_mock: Any,
+) -> None:
+    common_app_ingest(
+        pytestconfig=pytestconfig,
+        requests_mock=requests_mock,
+        output_mcp_path=f"{tmp_path}/powerbi_mces.json",
+        override_config={
+            "extract_app": True,
+        },
+    )
+
+    golden_file = "golden_test_app_ingest.json"
+
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=f"{tmp_path}/powerbi_mces.json",
+        golden_path=f"{test_resources_dir}/{golden_file}",
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
+@pytest.mark.integration
+def test_powerbi_app_ingest_info_message(
+    mock_msal: MagicMock,
+    pytestconfig: pytest.Config,
+    tmp_path: str,
+    mock_time: datetime.datetime,
+    requests_mock: Any,
+) -> None:
+    pipeline = common_app_ingest(
+        pytestconfig=pytestconfig,
+        requests_mock=requests_mock,
+        output_mcp_path=f"{tmp_path}/powerbi_mces.json",
+    )
+
+    assert isinstance(pipeline.source, PowerBiDashboardSource)  # to silent the lint
+
+    info_entries: dict = pipeline.source.reporter._structured_logs._entries.get(
+        StructuredLogLevel.INFO, {}
+    )  # type :ignore
+
+    is_entry_present: bool = False
+    # Printing INFO entries
+    for key, entry in info_entries.items():
+        if entry.title == "App Ingestion Is Disabled":
+            is_entry_present = True
+            break
+
+    assert (
+        is_entry_present
+    ), "The extract_app flag should be set to false by default. We need to keep this flag as false until all GMS instances are updated to the latest release."
diff --git a/metadata-ingestion/tests/integration/powerbi/test_profiling.py b/metadata-ingestion/tests/integration/powerbi/test_profiling.py
index 7955386de8940..4b48bed003b1e 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_profiling.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_profiling.py
@@ -112,21 +112,28 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
             "status_code": 200,
             "json": admin_datasets_response,
         },
-        "https://api.powerbi.com/v1.0/myorg/groups?%24top=1000&%24skip=0&%24filter=type+eq+%27Workspace%27": {
+        "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": {
             "method": "GET",
             "status_code": 200,
             "json": {
-                "@odata.count": 3,
                 "value": [
                     {
                         "id": "64ED5CAD-7C10-4684-8180-826122881108",
                         "isReadOnly": True,
                         "name": "demo-workspace",
                         "type": "Workspace",
+                        "state": "Active",
                     }
                 ],
             },
         },
+        "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "value": [],
+            },
+        },
         "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": {
             "method": "GET",
             "status_code": 200,
@@ -176,6 +183,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
                         "id": "64ED5CAD-7C10-4684-8180-826122881108",
                         "name": "demo-workspace",
                         "state": "Active",
+                        "type": "Workspace",
                         "datasets": [
                             {
                                 "id": "05169CD2-E713-41E6-9600-1D8066D95445",
diff --git a/metadata-ingestion/tests/integration/powerbi/test_stateful_ingestion.py b/metadata-ingestion/tests/integration/powerbi/test_stateful_ingestion.py
index 077b48ca177b5..84f7a87ce5d2d 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_stateful_ingestion.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_stateful_ingestion.py
@@ -23,27 +23,35 @@ def register_mock_api_state1(request_mock):
             "status_code": 403,
             "json": {},
         },
-        "https://api.powerbi.com/v1.0/myorg/groups": {
+        "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": {
             "method": "GET",
             "status_code": 200,
             "json": {
-                "@odata.count": 1,
                 "value": [
                     {
                         "id": "64ED5CAD-7C10-4684-8180-826122881108",
                         "isReadOnly": True,
                         "name": "Workspace 1",
                         "type": "Workspace",
+                        "state": "Active",
                     },
                     {
                         "id": "44444444-7C10-4684-8180-826122881108",
                         "isReadOnly": True,
                         "name": "Multi Workspace",
                         "type": "Workspace",
+                        "state": "Active",
                     },
                 ],
             },
         },
+        "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "value": [],
+            },
+        },
         "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": {
             "method": "GET",
             "status_code": 200,
@@ -114,7 +122,7 @@ def register_mock_api_state2(request_mock):
             "status_code": 403,
             "json": {},
         },
-        "https://api.powerbi.com/v1.0/myorg/groups": {
+        "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": {
             "method": "GET",
             "status_code": 200,
             "json": {
@@ -135,6 +143,13 @@ def register_mock_api_state2(request_mock):
                 ],
             },
         },
+        "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "value": [],
+            },
+        },
         "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": {
             "method": "GET",
             "status_code": 200,
diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py
index ee1aafb6cf32d..95f096cc3def3 100644
--- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py
+++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py
@@ -1011,7 +1011,6 @@ def default_config():
 def test_qlik_sense_ingest(
     pytestconfig, tmp_path, requests_mock, mock_websocket_send_request
 ):
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense"
 
     register_mock_api(request_mock=requests_mock)
@@ -1051,7 +1050,6 @@ def test_qlik_sense_ingest(
 def test_platform_instance_ingest(
     pytestconfig, tmp_path, requests_mock, mock_websocket_send_request
 ):
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense"
 
     register_mock_api(request_mock=requests_mock)
diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json
index 12bb7734f30a6..645e710309b0d 100644
--- a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json
+++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json
@@ -10,23 +10,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496006,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496008,
+        "lastObserved": 1732608523763,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -56,7 +40,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496007,
+        "lastObserved": 1732608523764,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -65,40 +49,30 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:sigma",
-            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+            "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496008,
+        "lastObserved": 1732608523764,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
-                },
-                {
-                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
-                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496010,
+        "lastObserved": 1732608523833,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -107,16 +81,15 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Sigma Dataset"
-            ]
+            "platform": "urn:li:dataPlatform:sigma",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496009,
+        "lastObserved": 1732608523764,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -125,40 +98,44 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "ownership",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:Shubham_Jagtap",
-                    "type": "DATAOWNER"
-                }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
+            "typeNames": [
+                "Sigma Dataset"
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496009,
+        "lastObserved": 1732608523765,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+                },
+                {
+                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496011,
+        "lastObserved": 1732608523835,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -167,14 +144,14 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+            "removed": false
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496012,
+        "lastObserved": 1732608523781,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -206,33 +183,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496012,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
-    "changeType": "UPSERT",
-    "aspectName": "ownership",
-    "aspect": {
-        "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:Shubham_Jagtap",
-                    "type": "DATAOWNER"
-                }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496201,
+        "lastObserved": 1732608523781,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -241,49 +192,26 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
-    "aspect": {
-        "json": {
-            "platform": "urn:li:dataPlatform:sigma",
-            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496013,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Sigma Workspace"
-            ]
+            "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496200,
+        "lastObserved": 1732608523782,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
-                },
                 {
                     "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
                     "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
@@ -291,22 +219,19 @@
                 {
                     "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
                     "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
-                },
-                {
-                    "id": "New Folder"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496015,
+        "lastObserved": 1732608523765,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
@@ -316,7 +241,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496200,
+        "lastObserved": 1732608523782,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -334,34 +259,51 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496014,
+        "lastObserved": 1732608523783,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "tags": [
+            "customProperties": {
+                "VizualizationType": "bar",
+                "type": "visualization"
+            },
+            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true",
+            "title": "Count of Profile Id by Status",
+            "description": "",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
                 {
-                    "tag": "urn:li:tag:Deprecated"
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496015,
+        "lastObserved": 1732608523833,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -380,14 +322,14 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496014,
+        "lastObserved": 1732608523765,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -396,57 +338,76 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496199,
+        "lastObserved": 1732608523784,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "containerProperties",
+    "aspectName": "dashboardInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "platform": "sigma",
-                "instance": "cloud_instance",
-                "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b"
-            },
-            "name": "Acryl Data",
-            "created": {
-                "time": 1710232264826
+                "path": "Acryl Data",
+                "latestVersion": "2"
             },
+            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
+            "title": "Acryl Workbook",
+            "description": "",
+            "charts": [],
+            "datasets": [],
+            "dashboards": [
+                {
+                    "sourceUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)"
+                },
+                {
+                    "sourceUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)"
+                }
+            ],
             "lastModified": {
-                "time": 1710232264826
+                "created": {
+                    "time": 1713188691477,
+                    "actor": "urn:li:corpuser:datahub"
+                },
+                "lastModified": {
+                    "time": 1713189117302,
+                    "actor": "urn:li:corpuser:datahub"
+                }
             }
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496199,
+        "lastObserved": 1732608523785,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Sigma Workbook"
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496053,
+        "lastObserved": 1732608523785,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -455,227 +416,81 @@
                 {
                     "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
                     "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+                },
+                {
+                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+                },
+                {
+                    "id": "New Folder"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496202,
+        "lastObserved": 1732608523783,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "ownership",
     "aspect": {
         "json": {
-            "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:Shubham_Jagtap",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496055,
+        "lastObserved": 1732608523786,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "inputFields",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "fields": [
+            "tags": [
                 {
-                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pk)",
-                    "schemaField": {
-                        "fieldPath": "Pk",
-                        "nullable": false,
-                        "type": {
-                            "type": {
-                                "com.linkedin.schema.StringType": {}
-                            }
-                        },
-                        "nativeDataType": "String",
-                        "recursive": false,
-                        "isPartOfKey": false
-                    }
-                },
-                {
-                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pet Fk)",
-                    "schemaField": {
-                        "fieldPath": "Pet Fk",
-                        "nullable": false,
-                        "type": {
-                            "type": {
-                                "com.linkedin.schema.StringType": {}
-                            }
-                        },
-                        "nativeDataType": "String",
-                        "recursive": false,
-                        "isPartOfKey": false
-                    }
-                },
-                {
-                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Human Fk)",
-                    "schemaField": {
-                        "fieldPath": "Human Fk",
-                        "nullable": false,
-                        "type": {
-                            "type": {
-                                "com.linkedin.schema.StringType": {}
-                            }
-                        },
-                        "nativeDataType": "String",
-                        "recursive": false,
-                        "isPartOfKey": false
-                    }
-                },
-                {
-                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Status)",
-                    "schemaField": {
-                        "fieldPath": "Status",
-                        "nullable": false,
-                        "type": {
-                            "type": {
-                                "com.linkedin.schema.StringType": {}
-                            }
-                        },
-                        "nativeDataType": "String",
-                        "recursive": false,
-                        "isPartOfKey": false
-                    }
-                },
-                {
-                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Created At)",
-                    "schemaField": {
-                        "fieldPath": "Created At",
-                        "nullable": false,
-                        "type": {
-                            "type": {
-                                "com.linkedin.schema.StringType": {}
-                            }
-                        },
-                        "nativeDataType": "String",
-                        "recursive": false,
-                        "isPartOfKey": false
-                    }
-                },
-                {
-                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Updated At)",
-                    "schemaField": {
-                        "fieldPath": "Updated At",
-                        "nullable": false,
-                        "type": {
-                            "type": {
-                                "com.linkedin.schema.StringType": {}
-                            }
-                        },
-                        "nativeDataType": "String",
-                        "recursive": false,
-                        "isPartOfKey": false
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496055,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
-    "changeType": "UPSERT",
-    "aspectName": "chartInfo",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "VizualizationType": "levelTable",
-                "type": "table"
-            },
-            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true",
-            "title": "ADOPTIONS",
-            "description": "",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718348049212,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
-                },
-                {
-                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
-                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
-                },
-                {
-                    "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-                    "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
+                    "tag": "urn:li:tag:Warning"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496058,
+        "lastObserved": 1732608523786,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "ownership",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:Shubham_Jagtap",
-                    "type": "DATAOWNER"
-                }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
+            "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496019,
+        "lastObserved": 1732608523786,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -691,7 +506,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496021,
+        "lastObserved": 1732608523787,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -713,6 +528,7 @@
                 "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)"
             ],
             "datasets": [],
+            "dashboards": [],
             "lastModified": {
                 "created": {
                     "time": 0,
@@ -726,87 +542,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496022,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-    "changeType": "UPSERT",
-    "aspectName": "containerProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "platform": "sigma",
-                "instance": "cloud_instance",
-                "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b",
-                "path": "Acryl Data",
-                "latestVersion": "2"
-            },
-            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
-            "name": "Acryl Workbook",
-            "created": {
-                "time": 1713188691477
-            },
-            "lastModified": {
-                "time": 1713189117302
-            }
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496016,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496023,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Sigma Workbook"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496018,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496017,
+        "lastObserved": 1732608523788,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -823,7 +559,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496023,
+        "lastObserved": 1732608523788,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -845,108 +581,87 @@
                     "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
                 },
                 {
-                    "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-                    "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496024,
+        "lastObserved": 1732608523788,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "tags": [
+            "path": [
                 {
-                    "tag": "urn:li:tag:Warning"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+                },
+                {
+                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+                },
+                {
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496019,
+        "lastObserved": 1732608523787,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-    "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
-    "aspect": {
-        "json": {
-            "platform": "urn:li:dataPlatform:sigma",
-            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496018,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496114,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
-                },
+            "tags": [
                 {
-                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
-                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+                    "tag": "urn:li:tag:Deprecated"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496020,
+        "lastObserved": 1732608523783,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "ownership",
     "aspect": {
         "json": {
-            "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:Shubham_Jagtap",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496116,
+        "lastObserved": 1732608523782,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1047,208 +762,20 @@
                         "nativeDataType": "String",
                         "recursive": false,
                         "isPartOfKey": false
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496117,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)",
-    "changeType": "UPSERT",
-    "aspectName": "chartInfo",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "VizualizationType": "bar",
-                "type": "visualization"
-            },
-            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true",
-            "title": "Count of Profile Id by Status",
-            "description": "",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718348049268,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
-                },
-                {
-                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
-                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
-                },
-                {
-                    "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-                    "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496119,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496124,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
-    "changeType": "UPSERT",
-    "aspectName": "dashboardInfo",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "ElementsCount": "1"
-            },
-            "title": "Page 2",
-            "description": "",
-            "charts": [
-                "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)"
-            ],
-            "datasets": [],
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            }
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496125,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496020,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713794496125,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)",
-    "changeType": "UPSERT",
-    "aspectName": "chartInfo",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "VizualizationType": "levelTable",
-                "type": "table"
-            },
-            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true",
-            "title": "PETS ADOPTIONS JOIN",
-            "description": "",
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
-                }
-            },
-            "inputs": [
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)"
-                },
-                {
-                    "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)"
+                    }
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718348049351,
+        "lastObserved": 1732608523834,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1257,69 +784,179 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496188,
+        "lastObserved": 1732608523803,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:sigma",
-            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+                },
+                {
+                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496126,
+        "lastObserved": 1732608523806,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "chartInfo",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
-                },
-                {
-                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
-                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+            "customProperties": {
+                "VizualizationType": "levelTable",
+                "type": "table"
+            },
+            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true",
+            "title": "ADOPTIONS",
+            "description": "",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
                 },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
                 {
-                    "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-                    "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496126,
+        "lastObserved": 1732608523804,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "inputFields",
     "aspect": {
         "json": {
-            "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pk)",
+                    "schemaField": {
+                        "fieldPath": "Pk",
+                        "nullable": false,
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "String",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pet Fk)",
+                    "schemaField": {
+                        "fieldPath": "Pet Fk",
+                        "nullable": false,
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "String",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Human Fk)",
+                    "schemaField": {
+                        "fieldPath": "Human Fk",
+                        "nullable": false,
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "String",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Status)",
+                    "schemaField": {
+                        "fieldPath": "Status",
+                        "nullable": false,
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "String",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Created At)",
+                    "schemaField": {
+                        "fieldPath": "Created At",
+                        "nullable": false,
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "String",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Updated At)",
+                    "schemaField": {
+                        "fieldPath": "Updated At",
+                        "nullable": false,
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "String",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496189,
+        "lastObserved": 1732608523804,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1483,40 +1120,219 @@
                     }
                 },
                 {
-                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)",
-                    "schemaField": {
-                        "fieldPath": "Updated At",
-                        "nullable": false,
-                        "type": {
-                            "type": {
-                                "com.linkedin.schema.StringType": {}
-                            }
-                        },
-                        "nativeDataType": "String",
-                        "recursive": false,
-                        "isPartOfKey": false
-                    }
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)",
+                    "schemaField": {
+                        "fieldPath": "Updated At",
+                        "nullable": false,
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "String",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)",
+                    "schemaField": {
+                        "fieldPath": "Count of Profile Id",
+                        "nullable": false,
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "String",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523836,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523838,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "ElementsCount": "1"
+            },
+            "title": "Page 2",
+            "description": "",
+            "charts": [
+                "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)"
+            ],
+            "datasets": [],
+            "dashboards": [],
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523838,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:sigma",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523839,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+                },
+                {
+                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
                 },
                 {
-                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)",
-                    "schemaField": {
-                        "fieldPath": "Count of Profile Id",
-                        "nullable": false,
-                        "type": {
-                            "type": {
-                                "com.linkedin.schema.StringType": {}
-                            }
-                        },
-                        "nativeDataType": "String",
-                        "recursive": false,
-                        "isPartOfKey": false
-                    }
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523839,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)",
+                    "type": "COPY"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496120,
+        "lastObserved": 1732608523874,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:Deprecated",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "Deprecated"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523874,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523872,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:Shubham_Jagtap",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523873,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1698,7 +1514,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496194,
+        "lastObserved": 1732608523870,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1707,27 +1523,54 @@
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
-                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523866,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)",
+    "changeType": "UPSERT",
+    "aspectName": "chartInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "VizualizationType": "levelTable",
+                "type": "table"
+            },
+            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true",
+            "title": "PETS ADOPTIONS JOIN",
+            "description": "",
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
                 },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "inputs": [
                 {
-                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
-                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)"
                 },
                 {
-                    "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8",
-                    "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8"
+                    "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496194,
+        "lastObserved": 1732608523866,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1909,64 +1752,134 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496190,
+        "lastObserved": 1732608523867,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "upstreams": [
+            "platform": "urn:li:dataPlatform:sigma",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523873,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)",
-                    "type": "COPY"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718348049380,
+        "lastObserved": 1732608523874,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "tag",
-    "entityUrn": "urn:li:tag:Warning",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)",
     "changeType": "UPSERT",
-    "aspectName": "tagKey",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "name": "Warning"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)"
+                },
+                {
+                    "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+                    "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523869,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "sigma",
+                "instance": "cloud_instance",
+                "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b"
+            },
+            "name": "Acryl Data",
+            "created": {
+                "time": 1710232264826
+            },
+            "lastModified": {
+                "time": 1710232264826
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732608523872,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Sigma Workspace"
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496203,
+        "lastObserved": 1732608523873,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "tag",
-    "entityUrn": "urn:li:tag:Deprecated",
+    "entityUrn": "urn:li:tag:Warning",
     "changeType": "UPSERT",
     "aspectName": "tagKey",
     "aspect": {
         "json": {
-            "name": "Deprecated"
+            "name": "Warning"
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713794496203,
+        "lastObserved": 1732608523875,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json
index f800cb19f8811..bb37e7029330b 100644
--- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json
+++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json
@@ -261,37 +261,8 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "containerProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "platform": "sigma",
-                "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b",
-                "path": "Acryl Data",
-                "latestVersion": "2"
-            },
-            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
-            "name": "Acryl Workbook",
-            "created": {
-                "time": 1713188691477
-            },
-            "lastModified": {
-                "time": 1713189117302
-            }
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619227,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -300,7 +271,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619228,
+        "lastObserved": 1732513099680,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -338,32 +309,70 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": [
+            "typeNames": [
+                "Sigma Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732513099681,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "path": "Acryl Data",
+                "latestVersion": "2"
+            },
+            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
+            "title": "Acryl Workbook",
+            "description": "",
+            "charts": [],
+            "datasets": [],
+            "dashboards": [
                 {
-                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
-                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+                    "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)"
                 },
                 {
-                    "id": "New Folder"
+                    "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)"
                 }
-            ]
+            ],
+            "lastModified": {
+                "created": {
+                    "time": 1713188691477,
+                    "actor": "urn:li:corpuser:datahub"
+                },
+                "lastModified": {
+                    "time": 1713189117302,
+                    "actor": "urn:li:corpuser:datahub"
+                }
+            }
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619226,
+        "lastObserved": 1732535135915,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -382,41 +391,51 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619229,
+        "lastObserved": 1732513099681,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Sigma Workbook"
+            "tags": [
+                {
+                    "tag": "urn:li:tag:Warning"
+                }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619229,
+        "lastObserved": 1732513099682,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:sigma"
+            "path": [
+                {
+                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
+                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+                },
+                {
+                    "id": "New Folder"
+                }
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619228,
+        "lastObserved": 1713795619226,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -457,79 +476,62 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
     "changeType": "UPSERT",
-    "aspectName": "dashboardInfo",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "ElementsCount": "2"
-            },
-            "title": "Page 1",
-            "description": "",
-            "charts": [
-                "urn:li:chart:(sigma,kH0MeihtGs)",
-                "urn:li:chart:(sigma,Ml9C5ezT5W)"
-            ],
-            "datasets": [],
-            "lastModified": {
-                "created": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
+            "path": [
+                {
+                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
+                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
                 },
-                "lastModified": {
-                    "time": 0,
-                    "actor": "urn:li:corpuser:unknown"
+                {
+                    "id": "Acryl Workbook"
                 }
-            }
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619233,
+        "lastObserved": 1732545848809,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "ownership",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "owners": [
-                {
-                    "owner": "urn:li:corpuser:Shubham_Jagtap",
-                    "type": "DATAOWNER"
-                }
-            ],
-            "ownerTypes": {},
-            "lastModified": {
-                "time": 0,
-                "actor": "urn:li:corpuser:unknown"
-            }
+            "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619224,
+        "lastObserved": 1732513099682,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "tags": [
+            "path": [
                 {
-                    "tag": "urn:li:tag:Warning"
+                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
+                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+                },
+                {
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619230,
+        "lastObserved": 1732545848807,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -538,14 +540,34 @@
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "dashboardInfo",
     "aspect": {
         "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+            "customProperties": {
+                "ElementsCount": "2"
+            },
+            "title": "Page 1",
+            "description": "",
+            "charts": [
+                "urn:li:chart:(sigma,kH0MeihtGs)",
+                "urn:li:chart:(sigma,Ml9C5ezT5W)"
+            ],
+            "datasets": [],
+            "dashboards": [],
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            }
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619234,
+        "lastObserved": 1713795619233,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -554,59 +576,44 @@
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "globalTags",
+    "aspectName": "ownership",
     "aspect": {
         "json": {
-            "tags": [
+            "owners": [
                 {
-                    "tag": "urn:li:tag:Deprecated"
+                    "owner": "urn:li:corpuser:Shubham_Jagtap",
+                    "type": "DATAOWNER"
                 }
-            ]
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619226,
+        "lastObserved": 1713795619224,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "globalTags",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
-                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
-                },
+            "tags": [
                 {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "tag": "urn:li:tag:Deprecated"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619234,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619231,
+        "lastObserved": 1713795619226,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -642,51 +649,13 @@
                     "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
                 },
                 {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619375,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
-                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619231,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619373,
+        "lastObserved": 1732545848872,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -883,6 +852,7 @@
                 "urn:li:chart:(sigma,tQJu5N1l81)"
             ],
             "datasets": [],
+            "dashboards": [],
             "lastModified": {
                 "created": {
                     "time": 0,
@@ -901,6 +871,30 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
+                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732545848877,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@@ -914,14 +908,13 @@
                     "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
                 },
                 {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619270,
+        "lastObserved": 1732545848829,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1179,54 +1172,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619382,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619267,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619449,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
@@ -1409,31 +1354,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f",
-                    "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
-                },
-                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1713795619383,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@@ -1554,14 +1474,13 @@
                     "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f"
                 },
                 {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1713795619453,
+        "lastObserved": 1732545848921,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json
index d6b702bdfd669..1ce671f09d776 100644
--- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json
+++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json
@@ -279,37 +279,24 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "containerProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {
-                "platform": "sigma",
-                "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b",
-                "path": "New Acryl Data",
-                "latestVersion": "2"
-            },
-            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
-            "name": "Acryl Workbook",
-            "created": {
-                "time": 1713188691477
-            },
-            "lastModified": {
-                "time": 1713189117302
-            }
+            "removed": false
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101680,
+        "lastObserved": 1732513100094,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -318,48 +305,76 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101680,
+        "lastObserved": 1718004101684,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:sigma"
+            "typeNames": [
+                "Sigma Workbook"
+            ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101681,
+        "lastObserved": 1732513100095,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "dashboardInfo",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Sigma Workbook"
-            ]
+            "customProperties": {
+                "path": "New Acryl Data",
+                "latestVersion": "2"
+            },
+            "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7",
+            "title": "Acryl Workbook",
+            "description": "",
+            "charts": [],
+            "datasets": [],
+            "dashboards": [
+                {
+                    "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)"
+                },
+                {
+                    "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+                    "destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)"
+                }
+            ],
+            "lastModified": {
+                "created": {
+                    "time": 1713188691477,
+                    "actor": "urn:li:corpuser:datahub"
+                },
+                "lastModified": {
+                    "time": 1713189117302,
+                    "actor": "urn:li:corpuser:datahub"
+                }
+            }
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101681,
+        "lastObserved": 1732535136409,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -378,14 +393,14 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101682,
+        "lastObserved": 1732513100096,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
     "aspectName": "globalTags",
     "aspect": {
@@ -398,60 +413,7 @@
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101683,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b83da80a4d444484521d9f7aca958742"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101683,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
-                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101684,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101684,
+        "lastObserved": 1732513100096,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -473,6 +435,7 @@
                 "urn:li:chart:(sigma,Ml9C5ezT5W)"
             ],
             "datasets": [],
+            "dashboards": [],
             "lastModified": {
                 "created": {
                     "time": 0,
@@ -491,47 +454,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101686,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
-                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
-                },
-                {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101686,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
@@ -582,17 +504,65 @@
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)",
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
+                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732545849249,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+            "container": "urn:li:container:b83da80a4d444484521d9f7aca958742"
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101689,
+        "lastObserved": 1732513100096,
+        "runId": "sigma-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b83da80a4d444484521d9f7aca958742",
+                    "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
+                },
+                {
+                    "id": "Acryl Workbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1732545849248,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -717,14 +687,13 @@
                     "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
                 },
                 {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101692,
+        "lastObserved": 1732545849252,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -778,22 +747,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101695,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)",
@@ -914,14 +867,13 @@
                     "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
                 },
                 {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101697,
+        "lastObserved": 1732545849255,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1155,6 +1107,7 @@
                 "urn:li:chart:(sigma,tQJu5N1l81)"
             ],
             "datasets": [],
+            "dashboards": [],
             "lastModified": {
                 "created": {
                     "time": 0,
@@ -1174,17 +1127,17 @@
     }
 },
 {
-    "entityType": "dashboard",
-    "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)",
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+            "removed": false
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101704,
+        "lastObserved": 1718004101706,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1202,30 +1155,13 @@
                     "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
                 },
                 {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101704,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101706,
+        "lastObserved": 1732545849260,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
@@ -1263,22 +1199,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1718004101708,
-        "runId": "sigma-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "chart",
     "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)",
@@ -1474,14 +1394,13 @@
                     "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742"
                 },
                 {
-                    "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02",
-                    "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02"
+                    "id": "Acryl Workbook"
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1718004101712,
+        "lastObserved": 1732545849264,
         "runId": "sigma-test",
         "lastRunId": "no-run-id-provided"
     }
diff --git a/metadata-ingestion/tests/integration/sigma/test_sigma.py b/metadata-ingestion/tests/integration/sigma/test_sigma.py
index 6c01bf6dc80fe..19fa1448fee59 100644
--- a/metadata-ingestion/tests/integration/sigma/test_sigma.py
+++ b/metadata-ingestion/tests/integration/sigma/test_sigma.py
@@ -420,7 +420,6 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
 
 @pytest.mark.integration
 def test_sigma_ingest(pytestconfig, tmp_path, requests_mock):
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma"
 
     register_mock_api(request_mock=requests_mock)
@@ -464,7 +463,6 @@ def test_sigma_ingest(pytestconfig, tmp_path, requests_mock):
 
 @pytest.mark.integration
 def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock):
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma"
 
     register_mock_api(request_mock=requests_mock)
@@ -510,7 +508,6 @@ def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock):
 
 @pytest.mark.integration
 def test_sigma_ingest_shared_entities(pytestconfig, tmp_path, requests_mock):
-
     test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma"
 
     override_data = {
diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py
index 8f45be96625a4..862d27186703a 100644
--- a/metadata-ingestion/tests/integration/snowflake/common.py
+++ b/metadata-ingestion/tests/integration/snowflake/common.py
@@ -14,6 +14,11 @@
 NUM_OPS = 10
 NUM_USAGE = 0
 
+
+def is_secure(view_idx):
+    return view_idx == 1
+
+
 FROZEN_TIME = "2022-06-07 17:00:00"
 large_sql_query = """WITH object_access_history AS
         (
@@ -247,9 +252,25 @@ def default_query_results(  # noqa: C901
                 "name": f"VIEW_{view_idx}",
                 "created_on": datetime(2021, 6, 8, 0, 0, 0, 0),
                 "comment": "Comment for View",
-                "text": f"create view view_{view_idx} as select * from table_{view_idx}",
+                "is_secure": "true" if is_secure(view_idx) else "false",
+                "text": (
+                    f"create view view_{view_idx} as select * from table_{view_idx}"
+                    if not is_secure(view_idx)
+                    else None
+                ),
+            }
+            for view_idx in range(1, num_views + 1)
+        ]
+    elif query == SnowflakeQuery.get_secure_view_definitions():
+        return [
+            {
+                "TABLE_CATALOG": "TEST_DB",
+                "TABLE_SCHEMA": "TEST_SCHEMA",
+                "TABLE_NAME": f"VIEW_{view_idx}",
+                "VIEW_DEFINITION": f"create view view_{view_idx} as select * from table_{view_idx}",
             }
             for view_idx in range(1, num_views + 1)
+            if is_secure(view_idx)
         ]
     elif query == SnowflakeQuery.columns_for_schema("TEST_SCHEMA", "TEST_DB"):
         return [
@@ -441,7 +462,6 @@ def default_query_results(  # noqa: C901
             include_column_lineage=True,
         ),
     ):
-
         return [
             {
                 "DOWNSTREAM_TABLE_NAME": f"TEST_DB.TEST_SCHEMA.TABLE_{op_idx}",
diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
index 4415b1ad3e515..48ec46af069ce 100644
--- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
+++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
@@ -490,7 +490,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
             "name": "TABLE_1",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
@@ -789,7 +791,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
             "name": "TABLE_2",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
@@ -1088,7 +1092,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
             "name": "TABLE_3",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
@@ -1387,7 +1393,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
             "name": "TABLE_4",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
@@ -1686,7 +1694,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/",
             "name": "TABLE_5",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5",
@@ -1985,7 +1995,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
             "name": "TABLE_6",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
@@ -2284,7 +2296,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
             "name": "TABLE_7",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
@@ -2583,7 +2597,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
             "name": "TABLE_8",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
@@ -2882,7 +2898,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
             "name": "TABLE_9",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
@@ -3181,7 +3199,9 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
+            "customProperties": {
+                "CLUSTERING_KEY": "LINEAR(COL_1)"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
             "name": "TABLE_10",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
@@ -3471,23 +3491,25 @@
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "customProperties": {},
+            "customProperties": {
+                "IS_SECURE": "true"
+            },
             "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/",
             "name": "VIEW_1",
             "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1",
             "description": "Comment for View",
             "created": {
-                "time": 1623103200000
+                "time": 1623090600000
             },
             "lastModified": {
-                "time": 1623103200000
+                "time": 1623090600000
             },
             "tags": []
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_12_18-10_16_09",
+        "runId": "snowflake-2024_12_16-15_30_20-649nax",
         "lastRunId": "no-run-id-provided"
     }
 },
diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
index d232ae710e891..f22cbd122361d 100644
--- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
+++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
@@ -138,27 +138,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
-            "name": "TABLE_3",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_3"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_3"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -567,27 +589,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/",
-            "name": "VIEW_1",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1",
-            "description": "Comment for View",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "VIEW_1"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for View"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.VIEW_1"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/IS_SECURE",
+                "value": "true"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ivthci",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -808,27 +852,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
-            "name": "TABLE_1",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_1"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_1"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -1473,27 +1539,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
-            "name": "TABLE_10",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_10"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_10"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -1712,54 +1800,98 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/",
-            "name": "TABLE_5",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_5"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_5"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
-            "name": "TABLE_2",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_2"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_2"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -2301,27 +2433,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
-            "name": "TABLE_6",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_6"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_6"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -2621,27 +2775,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
-            "name": "TABLE_7",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_7"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_7"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -2664,27 +2840,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
-            "name": "TABLE_4",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_4"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_4"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -3162,27 +3360,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
-            "name": "TABLE_8",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_8"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_8"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -3302,27 +3522,49 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
-            "name": "TABLE_9",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "TABLE_9"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for Table"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.TABLE_9"
+            },
+            {
+                "op": "add",
+                "path": "/customProperties/CLUSTERING_KEY",
+                "value": "LINEAR(COL_1)"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
@@ -3607,27 +3849,44 @@
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
-    "changeType": "UPSERT",
+    "changeType": "PATCH",
     "aspectName": "datasetProperties",
     "aspect": {
-        "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/",
-            "name": "VIEW_2",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2",
-            "description": "Comment for View",
-            "created": {
-                "time": 1623110400000
+        "json": [
+            {
+                "op": "add",
+                "path": "/name",
+                "value": "VIEW_2"
             },
-            "lastModified": {
-                "time": 1623110400000
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Comment for View"
             },
-            "tags": []
-        }
+            {
+                "op": "add",
+                "path": "/created",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "time": 1623110400000
+                }
+            },
+            {
+                "op": "add",
+                "path": "/qualifiedName",
+                "value": "TEST_DB.TEST_SCHEMA.VIEW_2"
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
+        "runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
         "lastRunId": "no-run-id-provided"
     }
 },
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
index ca694b02cff01..1d7470d24f768 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
@@ -187,7 +187,9 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
 
 
 @freeze_time(FROZEN_TIME)
-def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
+def test_snowflake_private_link_and_incremental_mcps(
+    pytestconfig, tmp_path, mock_time, mock_datahub_graph
+):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/snowflake"
 
     # Run the metadata ingestion pipeline.
@@ -218,6 +220,7 @@ def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_
                         include_usage_stats=False,
                         format_sql_queries=True,
                         incremental_lineage=False,
+                        incremental_properties=True,
                         include_operational_stats=False,
                         platform_instance="instance1",
                         start_time=datetime(2022, 6, 6, 0, 0, 0, 0).replace(
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py
new file mode 100644
index 0000000000000..82f5691bcee3d
--- /dev/null
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py
@@ -0,0 +1,24 @@
+import os
+from unittest.mock import patch
+
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.source.snowflake.snowflake_queries import SnowflakeQueriesSource
+
+
+@patch("snowflake.connector.connect")
+def test_source_close_cleans_tmp(snowflake_connect, tmp_path):
+    with patch("tempfile.tempdir", str(tmp_path)):
+        source = SnowflakeQueriesSource.create(
+            {
+                "connection": {
+                    "account_id": "ABC12345.ap-south-1.aws",
+                    "username": "TST_USR",
+                    "password": "TST_PWD",
+                }
+            },
+            PipelineContext("run-id"),
+        )
+        assert len(os.listdir(tmp_path)) > 0
+        # This closes QueriesExtractor which in turn closes SqlParsingAggregator
+        source.close()
+        assert len(os.listdir(tmp_path)) == 0
diff --git a/metadata-ingestion/tests/integration/sql_server/docker-compose.yml b/metadata-ingestion/tests/integration/sql_server/docker-compose.yml
index 1046321e4f720..aed70503903c0 100644
--- a/metadata-ingestion/tests/integration/sql_server/docker-compose.yml
+++ b/metadata-ingestion/tests/integration/sql_server/docker-compose.yml
@@ -1,7 +1,7 @@
 version: "3"
 services:
   testsqlserver:
-    image: "mcr.microsoft.com/mssql/server:latest"
+    image: "mcr.microsoft.com/mssql/server:2022-latest"
     platform: linux/amd64
     container_name: "testsqlserver"
     environment:
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
index 558548bfc7a69..72dcda25c1296 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
@@ -113,11 +113,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738",
+                "job_id": "c2d77890-83ba-435f-879b-1c77fa38dd47",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2024-09-16 15:59:53.077000",
-                "date_modified": "2024-09-16 15:59:53.217000",
+                "date_created": "2024-12-05 16:44:43.910000",
+                "date_modified": "2024-12-05 16:44:44.043000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -154,6 +154,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
@@ -231,10 +247,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -245,17 +266,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -341,10 +357,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -355,17 +376,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -451,10 +467,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -465,17 +486,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -561,10 +577,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -575,17 +596,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -671,10 +687,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -685,17 +706,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -781,10 +797,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -795,17 +816,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -891,10 +907,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -905,17 +926,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1001,10 +1017,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1015,17 +1036,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1111,10 +1127,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1125,17 +1146,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1217,22 +1233,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
@@ -1386,6 +1386,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
@@ -1459,22 +1475,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
@@ -1498,7 +1498,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -1515,7 +1515,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1525,14 +1525,13 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {},
-                        "name": "Items",
-                        "description": "Description for table Items of schema Foo.",
+                        "name": "age_dist",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "DemoData.Foo.Items",
+                        "schemaName": "DemoData.Foo.age_dist",
                         "platform": "urn:li:dataPlatform:mssql",
                         "version": 0,
                         "created": {
@@ -1551,7 +1550,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "ID",
+                                "fieldPath": "Age",
                                 "nullable": true,
                                 "type": {
                                     "type": {
@@ -1563,14 +1562,147 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "ItemName",
+                                "fieldPath": "Count",
                                 "nullable": true,
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Items",
+                        "description": "Description for table Items of schema Foo.",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "DemoData.Foo.Items",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ItemName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -1942,6 +2074,183 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n",
+                            "is_view": "True"
+                        },
+                        "name": "PersonsView",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "DemoData.Foo.PersonsView",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "LastName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "FirstName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataFlow",
     "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)",
@@ -1973,8 +2282,8 @@
                 "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2024-09-16 15:59:53.010000",
-                "date_modified": "2024-09-16 15:59:53.010000"
+                "date_created": "2024-12-05 16:44:43.800000",
+                "date_modified": "2024-12-05 16:44:43.800000"
             },
             "externalUrl": "",
             "name": "DemoData.Foo.Proc.With.SpecialChar",
@@ -1991,14 +2300,40 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)",
     "changeType": "UPSERT",
-    "aspectName": "dataJobInputOutput",
+    "aspectName": "dataJobInfo",
     "aspect": {
         "json": {
-            "inputDatasets": [],
-            "outputDatasets": [],
-            "inputDatajobs": []
+            "customProperties": {
+                "procedure_depends_on": "{'DemoData.Foo.age_dist': 'USER_TABLE', 'DemoData.Foo.Items': 'USER_TABLE', 'DemoData.Foo.Persons': 'USER_TABLE', 'DemoData.Foo.SalesReason': 'USER_TABLE'}",
+                "depending_on_procedure": "{}",
+                "code": "CREATE PROCEDURE [Foo].[NewProc]\n    AS\n    BEGIN\n        --insert into items table from salesreason table\n        insert into Foo.Items (ID, ItemName)\n        SELECT TempID, Name\n        FROM Foo.SalesReason;\n\n\n       IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n       BEGIN\n            -- Create and populate if table doesn't exist\n            SELECT Age, COUNT(*) as Count\n            INTO Foo.age_dist\n            FROM Foo.Persons\n            GROUP BY Age\n        END\n        ELSE\n        BEGIN\n            -- Update existing table\n            TRUNCATE TABLE Foo.age_dist;\n\n            INSERT INTO Foo.age_dist (Age, Count)\n            SELECT Age, COUNT(*) as Count\n            FROM Foo.Persons\n            GROUP BY Age\n        END\n\n        SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n        \n        UPDATE DemoData.Foo.Persons\n        SET Age = t.Age\n        FROM DemoData.Foo.Persons p\n        JOIN #TEMPTABLE t ON p.ID = t.ID\n\n    END\n",
+                "input parameters": "[]",
+                "date_created": "2024-12-05 16:44:43.803000",
+                "date_modified": "2024-12-05 16:44:43.803000"
+            },
+            "externalUrl": "",
+            "name": "DemoData.Foo.NewProc",
+            "type": {
+                "string": "MSSQL_STORED_PROCEDURE"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -2084,10 +2419,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2098,17 +2438,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -2194,10 +2529,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2208,17 +2548,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -2300,22 +2635,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
@@ -2425,6 +2744,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
@@ -2502,10 +2837,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2516,17 +2856,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
+    "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -2612,10 +2947,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2626,17 +2966,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
+    "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -2722,10 +3057,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2736,17 +3076,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
+    "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -2832,10 +3167,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2846,17 +3186,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
+    "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -2942,10 +3277,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2956,17 +3296,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
+    "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -3052,10 +3387,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -3066,17 +3406,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
+    "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -3162,10 +3497,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -3176,17 +3516,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
+    "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -3272,10 +3607,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -3286,17 +3626,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
+    "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -3382,10 +3717,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -3396,17 +3736,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
+    "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -3488,22 +3823,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
@@ -3673,17 +3992,10 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
     "changeType": "UPSERT",
-    "aspectName": "containerProperties",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "customProperties": {
-                "platform": "mssql",
-                "env": "PROD",
-                "database": "NewData",
-                "schema": "FooNew"
-            },
-            "name": "FooNew",
-            "env": "PROD"
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -3696,10 +4008,17 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "containerProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "FooNew"
+            },
+            "name": "FooNew",
+            "env": "PROD"
         }
     },
     "systemMetadata": {
@@ -3712,10 +4031,10 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -3728,12 +4047,10 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Schema"
-            ]
+            "platform": "urn:li:dataPlatform:mssql"
         }
     },
     "systemMetadata": {
@@ -3746,10 +4063,12 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "typeNames": [
+                "Schema"
+            ]
         }
     },
     "systemMetadata": {
@@ -4079,6 +4398,175 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:f721da08adde46586c0f113287cb60d1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "view_definition": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n",
+                            "is_view": "True"
+                        },
+                        "name": "View1",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "NewData.FooNew.View1",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "LastName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "FirstName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                },
+                {
+                    "id": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+                    "urn": "urn:li:container:f721da08adde46586c0f113287cb60d1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
@@ -4156,10 +4644,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -4170,17 +4663,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
+    "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -4266,10 +4754,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -4280,17 +4773,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
+    "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -4376,10 +4864,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -4389,16 +4882,69 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
+            "upstreams": [
                 {
-                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
-                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+                    "type": "VIEW"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),firstname)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),firstname)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),lastname)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),lastname)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
@@ -4441,6 +4987,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
index 161d40ea91d91..0df89ff1eb94d 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
@@ -113,11 +113,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738",
+                "job_id": "c2d77890-83ba-435f-879b-1c77fa38dd47",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2024-09-16 15:59:53.077000",
-                "date_modified": "2024-09-16 15:59:53.217000",
+                "date_created": "2024-12-05 16:44:43.910000",
+                "date_modified": "2024-12-05 16:44:44.043000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -154,6 +154,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
@@ -231,10 +247,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -245,17 +266,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -341,10 +357,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -355,17 +376,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -451,10 +467,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -465,17 +486,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -561,10 +577,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -575,17 +596,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -671,10 +687,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -685,17 +706,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -781,10 +797,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -795,17 +816,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -891,10 +907,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -905,17 +926,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1001,10 +1017,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1015,17 +1036,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1111,10 +1127,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1125,17 +1146,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1217,22 +1233,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
@@ -1386,6 +1386,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
@@ -1459,22 +1475,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
@@ -1498,7 +1498,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -1515,7 +1515,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1525,14 +1525,13 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {},
-                        "name": "Items",
-                        "description": "Description for table Items of schema Foo.",
+                        "name": "age_dist",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "DemoData.Foo.Items",
+                        "schemaName": "DemoData.Foo.age_dist",
                         "platform": "urn:li:dataPlatform:mssql",
                         "version": 0,
                         "created": {
@@ -1551,7 +1550,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "ID",
+                                "fieldPath": "Age",
                                 "nullable": true,
                                 "type": {
                                     "type": {
@@ -1563,14 +1562,147 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "ItemName",
+                                "fieldPath": "Count",
                                 "nullable": true,
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Items",
+                        "description": "Description for table Items of schema Foo.",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "DemoData.Foo.Items",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ItemName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -1942,6 +2074,183 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n",
+                            "is_view": "True"
+                        },
+                        "name": "PersonsView",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "DemoData.Foo.PersonsView",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "LastName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "FirstName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataFlow",
     "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)",
@@ -1973,8 +2282,8 @@
                 "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2024-09-16 15:59:53.010000",
-                "date_modified": "2024-09-16 15:59:53.010000"
+                "date_created": "2024-12-05 16:44:43.800000",
+                "date_modified": "2024-12-05 16:44:43.800000"
             },
             "externalUrl": "",
             "name": "DemoData.Foo.Proc.With.SpecialChar",
@@ -1990,15 +2299,13 @@
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
     "changeType": "UPSERT",
-    "aspectName": "dataJobInputOutput",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "inputDatasets": [],
-            "outputDatasets": [],
-            "inputDatajobs": []
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -2084,10 +2391,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2098,17 +2410,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -2194,10 +2501,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2208,17 +2520,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -2304,10 +2611,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2317,16 +2629,20 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
+            "upstreams": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+                    "type": "VIEW"
                 }
             ]
         }
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
index 161d40ea91d91..b36188405e7e1 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
@@ -1,13 +1,14 @@
 [
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+    "entityUrn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData"
             },
@@ -23,7 +24,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+    "entityUrn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -39,12 +40,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+    "entityUrn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -55,7 +57,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+    "entityUrn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -73,12 +75,17 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+    "entityUrn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "path": []
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -89,7 +96,7 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,my-instance.Weekly Demo Data Backup,PROD)",
     "changeType": "UPSERT",
     "aspectName": "dataFlowInfo",
     "aspect": {
@@ -105,19 +112,36 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,my-instance.Weekly Demo Data Backup,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738",
+                "job_id": "b8907be7-52f5-4df4-a870-f4fe0679ec45",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2024-09-16 15:59:53.077000",
-                "date_modified": "2024-09-16 15:59:53.217000",
+                "date_created": "2024-12-19 12:34:45.843000",
+                "date_modified": "2024-12-19 12:34:46.017000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -138,7 +162,24 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -156,13 +197,30 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:5726a09b23f60be6f661206c879a3683",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5726a09b23f60be6f661206c879a3683",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_accessadmin"
@@ -179,7 +237,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:5726a09b23f60be6f661206c879a3683",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -195,12 +253,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:5726a09b23f60be6f661206c879a3683",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -211,7 +270,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:5726a09b23f60be6f661206c879a3683",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -229,12 +288,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:5726a09b23f60be6f661206c879a3683",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -245,17 +313,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:5d8a64d9bc388814ac06d9a4d7a3ad22",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -266,13 +329,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:5d8a64d9bc388814ac06d9a4d7a3ad22",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_backupoperator"
@@ -289,7 +353,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:5d8a64d9bc388814ac06d9a4d7a3ad22",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -305,12 +369,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:5d8a64d9bc388814ac06d9a4d7a3ad22",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -321,7 +386,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:5d8a64d9bc388814ac06d9a4d7a3ad22",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -339,12 +404,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:5d8a64d9bc388814ac06d9a4d7a3ad22",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -355,17 +429,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:d5f6914a2b8e0dd461f1ad02e7b28c11",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -376,13 +445,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:d5f6914a2b8e0dd461f1ad02e7b28c11",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_datareader"
@@ -399,7 +469,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:d5f6914a2b8e0dd461f1ad02e7b28c11",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -415,12 +485,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:d5f6914a2b8e0dd461f1ad02e7b28c11",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -431,7 +502,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:d5f6914a2b8e0dd461f1ad02e7b28c11",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -449,12 +520,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:d5f6914a2b8e0dd461f1ad02e7b28c11",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -465,17 +545,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:e3f86c86f3794233740cad99cba0b854",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -486,13 +561,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:e3f86c86f3794233740cad99cba0b854",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_datawriter"
@@ -509,7 +585,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:e3f86c86f3794233740cad99cba0b854",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -525,12 +601,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:e3f86c86f3794233740cad99cba0b854",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -541,7 +618,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:e3f86c86f3794233740cad99cba0b854",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -559,12 +636,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:e3f86c86f3794233740cad99cba0b854",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -575,17 +661,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:c978c9ed6c196412685945ad89f8fbd6",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -596,13 +677,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:c978c9ed6c196412685945ad89f8fbd6",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_ddladmin"
@@ -619,7 +701,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:c978c9ed6c196412685945ad89f8fbd6",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -635,12 +717,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:c978c9ed6c196412685945ad89f8fbd6",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -651,7 +734,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:c978c9ed6c196412685945ad89f8fbd6",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -669,12 +752,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:c978c9ed6c196412685945ad89f8fbd6",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -685,17 +777,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:17749025f27ce9ebd6febcaa6a49d715",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -706,13 +793,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:17749025f27ce9ebd6febcaa6a49d715",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_denydatareader"
@@ -729,7 +817,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:17749025f27ce9ebd6febcaa6a49d715",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -745,12 +833,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:17749025f27ce9ebd6febcaa6a49d715",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -761,7 +850,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:17749025f27ce9ebd6febcaa6a49d715",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -779,12 +868,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:17749025f27ce9ebd6febcaa6a49d715",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -795,17 +893,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:63c0518620c06ef7af76019fea52b862",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -816,13 +909,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:63c0518620c06ef7af76019fea52b862",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_denydatawriter"
@@ -839,7 +933,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:63c0518620c06ef7af76019fea52b862",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -855,12 +949,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:63c0518620c06ef7af76019fea52b862",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -871,7 +966,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:63c0518620c06ef7af76019fea52b862",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -889,12 +984,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:63c0518620c06ef7af76019fea52b862",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -905,17 +1009,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:c6e96aed010f9205f809c1ce9a530003",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -926,13 +1025,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:c6e96aed010f9205f809c1ce9a530003",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_owner"
@@ -949,7 +1049,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:c6e96aed010f9205f809c1ce9a530003",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -965,12 +1065,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:c6e96aed010f9205f809c1ce9a530003",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -981,7 +1082,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:c6e96aed010f9205f809c1ce9a530003",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -999,12 +1100,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:c6e96aed010f9205f809c1ce9a530003",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1015,17 +1125,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:895216bb602fb0002beac82d96507acf",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -1036,13 +1141,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:895216bb602fb0002beac82d96507acf",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "db_securityadmin"
@@ -1059,7 +1165,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:895216bb602fb0002beac82d96507acf",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1075,12 +1181,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:895216bb602fb0002beac82d96507acf",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -1091,7 +1198,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:895216bb602fb0002beac82d96507acf",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1109,12 +1216,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:895216bb602fb0002beac82d96507acf",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1125,17 +1241,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:92899b29bb814fdeb1186eb99139073f",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -1146,13 +1257,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
+    "entityUrn": "urn:li:container:92899b29bb814fdeb1186eb99139073f",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "dbo"
@@ -1169,7 +1281,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
+    "entityUrn": "urn:li:container:92899b29bb814fdeb1186eb99139073f",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1185,12 +1297,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
+    "entityUrn": "urn:li:container:92899b29bb814fdeb1186eb99139073f",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -1201,7 +1314,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
+    "entityUrn": "urn:li:container:92899b29bb814fdeb1186eb99139073f",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1219,31 +1332,19 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
+    "entityUrn": "urn:li:container:92899b29bb814fdeb1186eb99139073f",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
                 }
             ]
         }
@@ -1256,12 +1357,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.dbo.Products,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1"
+            "container": "urn:li:container:92899b29bb814fdeb1186eb99139073f"
         }
     },
     "systemMetadata": {
@@ -1273,7 +1374,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.dbo.Products,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1345,7 +1446,24 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.dbo.Products,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.dbo.Products,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1363,19 +1481,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.dbo.Products,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
                 },
                 {
-                    "id": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
-                    "urn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1"
+                    "id": "urn:li:container:92899b29bb814fdeb1186eb99139073f",
+                    "urn": "urn:li:container:92899b29bb814fdeb1186eb99139073f"
                 }
             ]
         }
@@ -1388,13 +1510,30 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+    "entityUrn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "Foo"
@@ -1411,7 +1550,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+    "entityUrn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1427,12 +1566,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+    "entityUrn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -1443,7 +1583,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+    "entityUrn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1461,31 +1601,19 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+    "entityUrn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
                 }
             ]
         }
@@ -1498,12 +1626,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.age_dist,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+            "container": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
         }
     },
     "systemMetadata": {
@@ -1515,7 +1643,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.age_dist,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1525,14 +1653,13 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {},
-                        "name": "Items",
-                        "description": "Description for table Items of schema Foo.",
+                        "name": "age_dist",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "DemoData.Foo.Items",
+                        "schemaName": "DemoData.Foo.age_dist",
                         "platform": "urn:li:dataPlatform:mssql",
                         "version": 0,
                         "created": {
@@ -1551,7 +1678,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "ID",
+                                "fieldPath": "Age",
                                 "nullable": true,
                                 "type": {
                                     "type": {
@@ -1563,14 +1690,14 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "ItemName",
+                                "fieldPath": "Count",
                                 "nullable": true,
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "nativeDataType": "INTEGER",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -1588,14 +1715,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.age_dist,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Table"
-            ]
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -1606,20 +1732,196 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.age_dist,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                },
-                {
-                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
-                }
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.age_dist,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                },
+                {
+                    "id": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
+                    "urn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Items,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Items,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Items",
+                        "description": "Description for table Items of schema Foo.",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "DemoData.Foo.Items",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ItemName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Items,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Items,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Items,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                },
+                {
+                    "id": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
+                    "urn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
+                }
             ]
         }
     },
@@ -1631,12 +1933,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Persons,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+            "container": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
         }
     },
     "systemMetadata": {
@@ -1648,7 +1950,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Persons,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1745,7 +2047,24 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Persons,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Persons,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1763,19 +2082,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Persons,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
                 },
                 {
-                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                },
+                {
+                    "id": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
+                    "urn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
                 }
             ]
         }
@@ -1788,12 +2111,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.SalesReason,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+            "container": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
         }
     },
     "systemMetadata": {
@@ -1805,7 +2128,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.SalesReason,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1880,12 +2203,12 @@
                             {
                                 "name": "FK_TempSales_SalesReason",
                                 "foreignFields": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD),ID)"
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Persons,PROD),ID)"
                                 ],
                                 "sourceFields": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD),TempID)"
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.SalesReason,PROD),TempID)"
                                 ],
-                                "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)"
+                                "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.Persons,PROD)"
                             }
                         ]
                     }
@@ -1901,7 +2224,24 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.SalesReason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.SalesReason,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1919,19 +2259,221 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.SalesReason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                },
+                {
+                    "id": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
+                    "urn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.PersonsView,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "is_view": "True",
+                            "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n"
+                        },
+                        "name": "PersonsView",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "DemoData.Foo.PersonsView",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "LastName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "FirstName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.PersonsView,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.PersonsView,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
                 },
                 {
-                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                    "id": "urn:li:container:6fbadfb496ee98718da210cc2fca1680",
+                    "urn": "urn:li:container:6fbadfb496ee98718da210cc2fca1680"
                 }
             ]
         }
@@ -1944,7 +2486,7 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD)",
     "changeType": "UPSERT",
     "aspectName": "dataFlowInfo",
     "aspect": {
@@ -1960,9 +2502,26 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1973,8 +2532,8 @@
                 "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2024-09-16 15:59:53.010000",
-                "date_modified": "2024-09-16 15:59:53.010000"
+                "date_created": "2024-12-19 12:34:45.660000",
+                "date_modified": "2024-12-19 12:34:45.660000"
             },
             "externalUrl": "",
             "name": "DemoData.Foo.Proc.With.SpecialChar",
@@ -1991,14 +2550,58 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
-    "aspectName": "dataJobInputOutput",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "inputDatasets": [],
-            "outputDatasets": [],
-            "inputDatajobs": []
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),NewProc)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "procedure_depends_on": "{'DemoData.Foo.age_dist': 'USER_TABLE', 'DemoData.Foo.Items': 'USER_TABLE', 'DemoData.Foo.Persons': 'USER_TABLE', 'DemoData.Foo.SalesReason': 'USER_TABLE'}",
+                "depending_on_procedure": "{}",
+                "code": "CREATE PROCEDURE [Foo].[NewProc]\n    AS\n    BEGIN\n        --insert into items table from salesreason table\n        insert into Foo.Items (ID, ItemName)\n        SELECT TempID, Name\n        FROM Foo.SalesReason;\n\n\n       IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n       BEGIN\n            -- Create and populate if table doesn't exist\n            SELECT Age, COUNT(*) as Count\n            INTO Foo.age_dist\n            FROM Foo.Persons\n            GROUP BY Age\n        END\n        ELSE\n        BEGIN\n            -- Update existing table\n            TRUNCATE TABLE Foo.age_dist;\n\n            INSERT INTO Foo.age_dist (Age, Count)\n            SELECT Age, COUNT(*) as Count\n            FROM Foo.Persons\n            GROUP BY Age\n        END\n\n        SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n        \n        UPDATE DemoData.Foo.Persons\n        SET Age = t.Age\n        FROM DemoData.Foo.Persons p\n        JOIN #TEMPTABLE t ON p.ID = t.ID\n\n    END\n",
+                "input parameters": "[]",
+                "date_created": "2024-12-19 12:34:45.667000",
+                "date_modified": "2024-12-19 12:34:45.667000"
+            },
+            "externalUrl": "",
+            "name": "DemoData.Foo.NewProc",
+            "type": {
+                "string": "MSSQL_STORED_PROCEDURE"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),NewProc)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -2009,13 +2612,30 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:5631370915311469374ef3cb5f0ebbf0",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5631370915311469374ef3cb5f0ebbf0",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "guest"
@@ -2032,7 +2652,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:5631370915311469374ef3cb5f0ebbf0",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2048,12 +2668,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:5631370915311469374ef3cb5f0ebbf0",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -2064,7 +2685,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:5631370915311469374ef3cb5f0ebbf0",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2082,12 +2703,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:5631370915311469374ef3cb5f0ebbf0",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2098,17 +2728,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:63c0319e212536168ec5b7dce2b7da2f",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -2119,13 +2744,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:63c0319e212536168ec5b7dce2b7da2f",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "INFORMATION_SCHEMA"
@@ -2142,7 +2768,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:63c0319e212536168ec5b7dce2b7da2f",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2158,12 +2784,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:63c0319e212536168ec5b7dce2b7da2f",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -2174,7 +2801,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:63c0319e212536168ec5b7dce2b7da2f",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2192,12 +2819,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:63c0319e212536168ec5b7dce2b7da2f",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2208,17 +2844,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:b0e2ef63fa03ab69f77b60844124ec97",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
         }
     },
     "systemMetadata": {
@@ -2229,13 +2860,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:b0e2ef63fa03ab69f77b60844124ec97",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "mssql",
+                "instance": "my-instance",
                 "env": "PROD",
                 "database": "DemoData",
                 "schema": "sys"
@@ -2252,7 +2884,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:b0e2ef63fa03ab69f77b60844124ec97",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2268,12 +2900,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:b0e2ef63fa03ab69f77b60844124ec97",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:mssql"
+            "platform": "urn:li:dataPlatform:mssql",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
         }
     },
     "systemMetadata": {
@@ -2284,7 +2917,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:b0e2ef63fa03ab69f77b60844124ec97",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2302,12 +2935,21 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:b0e2ef63fa03ab69f77b60844124ec97",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mssql,my-instance)"
+                },
+                {
+                    "id": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a",
+                    "urn": "urn:li:container:db8117ee3cc6397c503e7824ae3e0f6a"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2317,16 +2959,20 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.PersonsView,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
+            "upstreams": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.demodata.foo.persons,PROD)",
+                    "type": "VIEW"
                 }
             ]
         }
@@ -2339,7 +2985,7 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2355,7 +3001,23 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,my-instance.Weekly Demo Data Backup,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),NewProc)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2371,7 +3033,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2387,7 +3049,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
index 29124f6fc156c..ebcadcc11dcbf 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
@@ -113,11 +113,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738",
+                "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2024-09-16 15:59:53.077000",
-                "date_modified": "2024-09-16 15:59:53.217000",
+                "date_created": "2024-11-22 12:58:03.260000",
+                "date_modified": "2024-11-22 12:58:03.440000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -154,6 +154,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
@@ -231,10 +247,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -245,17 +266,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02",
+    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -341,10 +357,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -355,17 +376,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820",
+    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -451,10 +467,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -465,17 +486,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec",
+    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -561,10 +577,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -575,17 +596,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8",
+    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -671,10 +687,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -685,17 +706,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5",
+    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -781,10 +797,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -795,17 +816,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f",
+    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -891,10 +907,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -905,17 +926,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7",
+    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1001,10 +1017,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1015,17 +1036,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47",
+    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1111,10 +1127,15 @@
     "entityType": "container",
     "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1125,17 +1146,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619",
+    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
         }
     },
     "systemMetadata": {
@@ -1217,22 +1233,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1",
@@ -1386,6 +1386,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
@@ -1459,22 +1475,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
@@ -1498,7 +1498,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -1515,7 +1515,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1525,14 +1525,13 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {},
-                        "name": "Items",
-                        "description": "Description for table Items of schema Foo.",
+                        "name": "age_dist",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "demodata.foo.items",
+                        "schemaName": "demodata.foo.age_dist",
                         "platform": "urn:li:dataPlatform:mssql",
                         "version": 0,
                         "created": {
@@ -1551,7 +1550,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "ID",
+                                "fieldPath": "Age",
                                 "nullable": true,
                                 "type": {
                                     "type": {
@@ -1563,14 +1562,14 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "ItemName",
+                                "fieldPath": "Count",
                                 "nullable": true,
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "nativeDataType": "INTEGER",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -1586,10 +1585,2624 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Items",
+                        "description": "Description for table Items of schema Foo.",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "demodata.foo.items",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ItemName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
     "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Persons",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "demodata.foo.persons",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            },
+                            {
+                                "fieldPath": "LastName",
+                                "nullable": false,
+                                "description": "Description for column LastName of table Persons of schema Foo.",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "FirstName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "SalesReason",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "demodata.foo.salesreason",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "TempID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            },
+                            {
+                                "fieldPath": "SomeId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNIQUEIDENTIFIER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ],
+                        "foreignKeys": [
+                            {
+                                "name": "FK_TempSales_SalesReason",
+                                "foreignFields": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)"
+                                ],
+                                "sourceFields": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),TempID)"
+                                ],
+                                "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n",
+                            "is_view": "True"
+                        },
+                        "name": "PersonsView",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "demodata.foo.personsview",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "LastName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "FirstName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                },
+                {
+                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
+                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataFlowInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "",
+            "name": "DemoData.Foo.stored_procedures"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "procedure_depends_on": "{}",
+                "depending_on_procedure": "{}",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "input parameters": "['@ID']",
+                "parameter @ID": "{'type': 'int'}",
+                "date_created": "2024-11-22 12:58:03.137000",
+                "date_modified": "2024-11-22 12:58:03.137000"
+            },
+            "externalUrl": "",
+            "name": "DemoData.Foo.Proc.With.SpecialChar",
+            "type": {
+                "string": "MSSQL_STORED_PROCEDURE"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "procedure_depends_on": "{'DemoData.Foo.age_dist': 'USER_TABLE', 'DemoData.Foo.Items': 'USER_TABLE', 'DemoData.Foo.Persons': 'USER_TABLE', 'DemoData.Foo.SalesReason': 'USER_TABLE'}",
+                "depending_on_procedure": "{}",
+                "code": "CREATE PROCEDURE [Foo].[NewProc]\n    AS\n    BEGIN\n        --insert into items table from salesreason table\n        insert into Foo.Items (ID, ItemName)\n        SELECT TempID, Name\n        FROM Foo.SalesReason;\n\n\n       IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n       BEGIN\n            -- Create and populate if table doesn't exist\n            SELECT Age, COUNT(*) as Count\n            INTO Foo.age_dist\n            FROM Foo.Persons\n            GROUP BY Age\n        END\n        ELSE\n        BEGIN\n            -- Update existing table\n            TRUNCATE TABLE Foo.age_dist;\n\n            INSERT INTO Foo.age_dist (Age, Count)\n            SELECT Age, COUNT(*) as Count\n            FROM Foo.Persons\n            GROUP BY Age\n        END\n\n        SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n        \n        UPDATE DemoData.Foo.Persons\n        SET Age = t.Age\n        FROM DemoData.Foo.Persons p\n        JOIN #TEMPTABLE t ON p.ID = t.ID\n\n    END\n",
+                "input parameters": "[]",
+                "date_created": "2024-11-22 12:58:03.140000",
+                "date_modified": "2024-11-22 12:58:03.140000"
+            },
+            "externalUrl": "",
+            "name": "DemoData.Foo.NewProc",
+            "type": {
+                "string": "MSSQL_STORED_PROCEDURE"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "DemoData",
+                "schema": "guest"
+            },
+            "name": "guest",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "DemoData",
+                "schema": "INFORMATION_SCHEMA"
+            },
+            "name": "INFORMATION_SCHEMA",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "DemoData",
+                "schema": "sys"
+            },
+            "name": "sys",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
+                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData"
+            },
+            "name": "NewData",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Database"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_accessadmin"
+            },
+            "name": "db_accessadmin",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_backupoperator"
+            },
+            "name": "db_backupoperator",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_datareader"
+            },
+            "name": "db_datareader",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_datawriter"
+            },
+            "name": "db_datawriter",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_ddladmin"
+            },
+            "name": "db_ddladmin",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_denydatareader"
+            },
+            "name": "db_denydatareader",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_denydatawriter"
+            },
+            "name": "db_denydatawriter",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_owner"
+            },
+            "name": "db_owner",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "db_securityadmin"
+            },
+            "name": "db_securityadmin",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "dbo"
+            },
+            "name": "dbo",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.dbo.productsnew,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:269d0067d130eda0399a534fc787054c"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.dbo.productsnew,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "ProductsNew",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "newdata.dbo.productsnew",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ProductName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "MONEY",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.dbo.productsnew,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.dbo.productsnew,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                },
+                {
+                    "id": "urn:li:container:269d0067d130eda0399a534fc787054c",
+                    "urn": "urn:li:container:269d0067d130eda0399a534fc787054c"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "mssql",
+                "env": "PROD",
+                "database": "NewData",
+                "schema": "FooNew"
+            },
+            "name": "FooNew",
+            "env": "PROD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mssql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Schema"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.itemsnew,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:f721da08adde46586c0f113287cb60d1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.itemsnew,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "ItemsNew",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "newdata.foonew.itemsnew",
+                        "platform": "urn:li:dataPlatform:mssql",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "ID",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ItemName",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "SMALLMONEY",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.itemsnew,PROD)",
+    "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
@@ -1606,19 +4219,19 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.itemsnew,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
                 },
                 {
-                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                    "id": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+                    "urn": "urn:li:container:f721da08adde46586c0f113287cb60d1"
                 }
             ]
         }
@@ -1631,12 +4244,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+            "container": "urn:li:container:f721da08adde46586c0f113287cb60d1"
         }
     },
     "systemMetadata": {
@@ -1648,7 +4261,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1658,13 +4271,13 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {},
-                        "name": "Persons",
+                        "name": "PersonsNew",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "demodata.foo.persons",
+                        "schemaName": "newdata.foonew.personsnew",
                         "platform": "urn:li:dataPlatform:mssql",
                         "version": 0,
                         "created": {
@@ -1697,7 +4310,6 @@
                             {
                                 "fieldPath": "LastName",
                                 "nullable": false,
-                                "description": "Description for column LastName of table Persons of schema Foo.",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
@@ -1745,7 +4357,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1763,19 +4375,19 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
                 },
                 {
-                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+                    "id": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+                    "urn": "urn:li:container:f721da08adde46586c0f113287cb60d1"
                 }
             ]
         }
@@ -1788,12 +4400,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
+            "container": "urn:li:container:f721da08adde46586c0f113287cb60d1"
         }
     },
     "systemMetadata": {
@@ -1805,7 +4417,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1814,14 +4426,17 @@
                 },
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {},
-                        "name": "SalesReason",
+                        "customProperties": {
+                            "view_definition": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n",
+                            "is_view": "True"
+                        },
+                        "name": "View1",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "demodata.foo.salesreason",
+                        "schemaName": "newdata.foonew.view1",
                         "platform": "urn:li:dataPlatform:mssql",
                         "version": 0,
                         "created": {
@@ -1840,53 +4455,29 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "TempID",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER",
-                                "recursive": false,
-                                "isPartOfKey": true
-                            },
-                            {
-                                "fieldPath": "SomeId",
+                                "fieldPath": "LastName",
                                 "nullable": false,
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "UNIQUEIDENTIFIER",
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "Name",
+                                "fieldPath": "FirstName",
                                 "nullable": true,
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS",
+                                "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
-                        ],
-                        "foreignKeys": [
-                            {
-                                "name": "FK_TempSales_SalesReason",
-                                "foreignFields": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)"
-                                ],
-                                "sourceFields": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),TempID)"
-                                ],
-                                "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)"
-                            }
                         ]
                     }
                 }
@@ -1901,13 +4492,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "Table"
+                "View"
             ]
         }
     },
@@ -1919,39 +4510,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                },
-                {
-                    "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9",
-                    "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "dataFlowInfo",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "",
-            "name": "DemoData.Foo.stored_procedures"
+            "materialized": false,
+            "viewLogic": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n",
+            "viewLanguage": "SQL"
         }
     },
     "systemMetadata": {
@@ -1961,26 +4527,22 @@
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "dataJobInfo",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "procedure_depends_on": "{}",
-                "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
-                "input parameters": "['@ID']",
-                "parameter @ID": "{'type': 'int'}",
-                "date_created": "2024-09-16 15:59:53.010000",
-                "date_modified": "2024-09-16 15:59:53.010000"
-            },
-            "externalUrl": "",
-            "name": "DemoData.Foo.Proc.With.SpecialChar",
-            "type": {
-                "string": "MSSQL_STORED_PROCEDURE"
-            }
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                },
+                {
+                    "id": "urn:li:container:f721da08adde46586c0f113287cb60d1",
+                    "urn": "urn:li:container:f721da08adde46586c0f113287cb60d1"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1990,15 +4552,13 @@
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
     "changeType": "UPSERT",
-    "aspectName": "dataJobInputOutput",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "inputDatasets": [],
-            "outputDatasets": [],
-            "inputDatajobs": []
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -2009,7 +4569,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
@@ -2017,7 +4577,7 @@
             "customProperties": {
                 "platform": "mssql",
                 "env": "PROD",
-                "database": "DemoData",
+                "database": "NewData",
                 "schema": "guest"
             },
             "name": "guest",
@@ -2032,7 +4592,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2048,7 +4608,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
@@ -2064,7 +4624,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2082,12 +4642,17 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2098,17 +4663,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975",
+    "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -2119,7 +4679,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
@@ -2127,7 +4687,7 @@
             "customProperties": {
                 "platform": "mssql",
                 "env": "PROD",
-                "database": "DemoData",
+                "database": "NewData",
                 "schema": "INFORMATION_SCHEMA"
             },
             "name": "INFORMATION_SCHEMA",
@@ -2142,7 +4702,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2158,7 +4718,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
@@ -2174,7 +4734,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2192,12 +4752,17 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2208,17 +4773,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f",
+    "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
-                }
-            ]
+            "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
         }
     },
     "systemMetadata": {
@@ -2229,7 +4789,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
@@ -2237,7 +4797,7 @@
             "customProperties": {
                 "platform": "mssql",
                 "env": "PROD",
-                "database": "DemoData",
+                "database": "NewData",
                 "schema": "sys"
             },
             "name": "sys",
@@ -2252,7 +4812,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2268,7 +4828,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
@@ -2284,7 +4844,7 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2302,12 +4862,17 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+            "path": [
+                {
+                    "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5",
+                    "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2317,16 +4882,186 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
+            "upstreams": [
                 {
-                    "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100",
-                    "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),FirstName)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),FirstName)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),LastName)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),LastName)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)"
+            ],
+            "outputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)"
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD),Age)"
+                    ],
+                    "confidenceScore": 0.9
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),TempID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),tempid)"
+                    ],
+                    "confidenceScore": 0.9
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),name)"
+                    ],
+                    "confidenceScore": 0.9
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),Age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)"
+                    ],
+                    "confidenceScore": 0.35
                 }
             ]
         }
@@ -2369,6 +5104,22 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dataJob",
     "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/DemoData.Foo.NewProc.json b/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/DemoData.Foo.NewProc.json
new file mode 100644
index 0000000000000..609e3a6f42945
--- /dev/null
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/DemoData.Foo.NewProc.json
@@ -0,0 +1,57 @@
+[
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)"
+            ],
+            "outputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)"
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD),age)"
+                    ],
+                    "confidenceScore": 0.2
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),tempid)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),tempid)"
+                    ],
+                    "confidenceScore": 0.2
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),name)"
+                    ],
+                    "confidenceScore": 0.2
+                }
+            ]
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/demodata.foo.proc2.json b/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/demodata.foo.proc2.json
new file mode 100644
index 0000000000000..8ebd1c065ebf9
--- /dev/null
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/demodata.foo.proc2.json
@@ -0,0 +1,57 @@
+[
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.foo.stored_procedures,PROD),proc2)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)"
+            ],
+            "outputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)"
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),age)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD),age)"
+                    ],
+                    "confidenceScore": 0.2
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),tempid)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),tempid)"
+                    ],
+                    "confidenceScore": 0.2
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),name)"
+                    ],
+                    "confidenceScore": 0.2
+                }
+            ]
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/procedures/DemoData.Foo.NewProc.sql b/metadata-ingestion/tests/integration/sql_server/procedures/DemoData.Foo.NewProc.sql
new file mode 100644
index 0000000000000..52a8d1327653b
--- /dev/null
+++ b/metadata-ingestion/tests/integration/sql_server/procedures/DemoData.Foo.NewProc.sql
@@ -0,0 +1,37 @@
+CREATE PROCEDURE [Foo].[NewProc]
+    AS
+    BEGIN
+        --insert into items table from salesreason table
+        insert into Foo.Items (ID, ItemName)
+        SELECT TempID, Name
+        FROM Foo.SalesReason;
+
+
+       IF OBJECT_ID('Foo.age_dist', 'U') IS NULL
+
+       BEGIN
+            -- Create and populate if table doesn't exist
+            SELECT Age, COUNT(*) as Count
+            INTO Foo.age_dist
+            FROM Foo.Persons
+            GROUP BY Age
+        END
+        ELSE
+        BEGIN
+            -- Update existing table
+            TRUNCATE TABLE Foo.age_dist;
+
+            INSERT INTO Foo.age_dist (Age, Count)
+            SELECT Age, COUNT(*) as Count
+            FROM Foo.Persons
+            GROUP BY Age
+        END
+
+        SELECT * INTO #TempTable FROM NewData.FooNew.PersonsNew
+        
+        UPDATE DemoData.Foo.Persons
+        SET Age = t.Age
+        FROM DemoData.Foo.Persons p
+        JOIN #TempTable t ON p.ID = t.ID
+
+    END
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/procedures/demodata.foo.proc2.sql b/metadata-ingestion/tests/integration/sql_server/procedures/demodata.foo.proc2.sql
new file mode 100644
index 0000000000000..69194a8d2c546
--- /dev/null
+++ b/metadata-ingestion/tests/integration/sql_server/procedures/demodata.foo.proc2.sql
@@ -0,0 +1,36 @@
+CREATE PROCEDURE [foo].[proc2]
+    AS
+    BEGIN
+        --insert into items table from salesreason table
+        insert into foo.items (id, itemame)
+        SELECT tempid, name
+        FROM foo.salesreason;
+
+
+       IF OBJECT_ID('foo.age_dist', 'U') IS NULL
+
+       BEGIN
+            -- Create and populate if table doesn't exist
+            SELECT age, COUNT(*) as count
+            INTO foo.age_dist
+            FROM foo.persons
+            GROUP BY age
+        END
+        ELSE
+        BEGIN
+            -- Update existing table
+            TRUNCATE TABLE foo.age_dist;
+
+            INSERT INTO foo.age_dist (age, count)
+            SELECT age, COUNT(*) as count
+            FROM foo.persons
+            GROUP BY age
+        END
+
+        SELECT * INTO #temptable FROM newdata.foonew.personsnew
+        
+        UPDATE demodata.foo.persons
+        SET age = t.age
+        FROM demodata.foo.persons p
+        JOIN #temptable t ON p.ID = t.ID
+    END
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
index 77ecabc5a3fff..0c3c7ee2fd29e 100644
--- a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
+++ b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
@@ -1,3 +1,4 @@
+DROP DATABASE IF EXISTS NewData;
 CREATE DATABASE NewData;
 GO
 USE NewData;
@@ -14,7 +15,14 @@ CREATE TABLE FooNew.PersonsNew (
     FirstName varchar(255),
     Age int
 );
+GO
+CREATE VIEW FooNew.View1 AS
+SELECT LastName, FirstName
+FROM FooNew.PersonsNew
+WHERE Age > 18
+GO
 
+DROP DATABASE IF EXISTS DemoData;
 CREATE DATABASE DemoData;
 GO
 USE DemoData;
@@ -32,6 +40,8 @@ CREATE TABLE Foo.Persons (
     Age int
 );
 GO
+CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;
+GO
 CREATE TABLE Foo.SalesReason 
    (
       TempID int NOT NULL,
@@ -45,11 +55,54 @@ CREATE TABLE Foo.SalesReason
    )
 ;
 GO
+DROP PROCEDURE IF EXISTS [Foo].[Proc.With.SpecialChar];
+GO
 CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT
 AS
     SELECT @ID AS ThatDB;
 GO
 
+DROP PROCEDURE IF EXISTS [Foo].[NewProc];
+GO
+CREATE PROCEDURE [Foo].[NewProc]
+    AS
+    BEGIN
+        --insert into items table from salesreason table
+        insert into Foo.Items (ID, ItemName)
+        SELECT TempID, Name
+        FROM Foo.SalesReason;
+
+
+       IF OBJECT_ID('Foo.age_dist', 'U') IS NULL
+       BEGIN
+            -- Create and populate if table doesn't exist
+            SELECT Age, COUNT(*) as Count
+            INTO Foo.age_dist
+            FROM Foo.Persons
+            GROUP BY Age
+        END
+        ELSE
+        BEGIN
+            -- Update existing table
+            TRUNCATE TABLE Foo.age_dist;
+
+            INSERT INTO Foo.age_dist (Age, Count)
+            SELECT Age, COUNT(*) as Count
+            FROM Foo.Persons
+            GROUP BY Age
+        END
+
+        SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew
+        
+        UPDATE DemoData.Foo.Persons
+        SET Age = t.Age
+        FROM DemoData.Foo.Persons p
+        JOIN #TEMPTABLE t ON p.ID = t.ID
+
+    END
+GO
+
+EXEC Foo.NewProc
 GO
 EXEC sys.sp_addextendedproperty
 @name = N'MS_Description',
@@ -91,4 +144,4 @@ EXEC sp_attach_schedule
 GO
 EXEC dbo.sp_add_jobserver
     @job_name = N'Weekly Demo Data Backup'
-GO
+GO
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_no_db_with_filter.yml b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_no_db_with_filter.yml
index 3749499074adf..703f60b277b87 100644
--- a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_no_db_with_filter.yml
+++ b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_no_db_with_filter.yml
@@ -9,6 +9,9 @@ source:
     database_pattern:
       deny:
         - NewData
+    procedure_pattern:
+      deny:
+        - DemoData.Foo.NewProc
 
 sink:
   type: file
diff --git a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_to_file.yml b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_to_file.yml
index 40bef3ff104a3..e003ec39cd528 100644
--- a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_to_file.yml
+++ b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_to_file.yml
@@ -7,6 +7,7 @@ source:
     password: test!Password
     database: DemoData
     host_port: localhost:21433
+    platform_instance: my-instance
     # use_odbc: True
     # uri_args:
     #   driver: "ODBC Driver 17 for SQL Server"
diff --git a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_with_lower_case_urn.yml b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_with_lower_case_urn.yml
index ff1179034833f..94128810f026b 100644
--- a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_with_lower_case_urn.yml
+++ b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_with_lower_case_urn.yml
@@ -5,7 +5,6 @@ source:
   config:
     username: sa
     password: test!Password
-    database: DemoData
     host_port: localhost:21433
     convert_urns_to_lowercase: true
     # use_odbc: True
diff --git a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py
index 1f418ffbd32ea..b969f77b4c3c1 100644
--- a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py
+++ b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py
@@ -1,9 +1,16 @@
 import os
+import pathlib
 import subprocess
 import time
+from pathlib import Path
 
 import pytest
 
+from datahub.ingestion.source.sql.mssql.job_models import StoredProcedure
+from datahub.ingestion.source.sql.mssql.stored_procedure_lineage import (
+    generate_procedure_lineage,
+)
+from datahub.sql_parsing.schema_resolver import SchemaResolver
 from tests.test_helpers import mce_helpers
 from tests.test_helpers.click_helpers import run_datahub_cmd
 from tests.test_helpers.docker_helpers import cleanup_image, wait_for_port
@@ -57,3 +64,50 @@ def test_mssql_ingest(mssql_runner, pytestconfig, tmp_path, mock_time, config_fi
             r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['date_modified'\]",
         ],
     )
+
+
+PROCEDURE_SQLS_DIR = pathlib.Path(__file__).parent / "procedures"
+PROCEDURES_GOLDEN_DIR = pathlib.Path(__file__).parent / "golden_files/procedures/"
+procedure_sqls = [sql_file.name for sql_file in PROCEDURE_SQLS_DIR.iterdir()]
+
+
+@pytest.mark.parametrize("procedure_sql_file", procedure_sqls)
+@pytest.mark.integration
+def test_stored_procedure_lineage(
+    pytestconfig: pytest.Config, procedure_sql_file: str
+) -> None:
+    sql_file_path = PROCEDURE_SQLS_DIR / procedure_sql_file
+    procedure_code = sql_file_path.read_text()
+
+    # Procedure file is named as <db>.<schema>.<procedure_name>
+    splits = procedure_sql_file.split(".")
+    db = splits[0]
+    schema = splits[1]
+    name = splits[2]
+
+    procedure = StoredProcedure(
+        db=db,
+        schema=schema,
+        name=name,
+        flow=None,  # type: ignore # flow is not used in this test
+        code=procedure_code,
+    )
+    data_job_urn = f"urn:li:dataJob:(urn:li:dataFlow:(mssql,{db}.{schema}.stored_procedures,PROD),{name})"
+
+    schema_resolver = SchemaResolver(platform="mssql")
+
+    mcps = list(
+        generate_procedure_lineage(
+            schema_resolver=schema_resolver,
+            procedure=procedure,
+            procedure_job_urn=data_job_urn,
+            is_temp_table=lambda name: "temp" in name.lower(),
+        )
+    )
+    mce_helpers.check_goldens_stream(
+        pytestconfig,
+        outputs=mcps,
+        golden_path=(
+            PROCEDURES_GOLDEN_DIR / Path(procedure_sql_file).with_suffix(".json")
+        ),
+    )
diff --git a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json
index 767b85a72b975..4801af9465f2c 100644
--- a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json
+++ b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json
@@ -26,6 +26,7 @@
                             "urn:li:chart:(superset,11)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 0,
@@ -73,6 +74,7 @@
                             "urn:li:chart:(superset,13)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 0,
diff --git a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json
index 0f20799017979..ac6a3b6942a32 100644
--- a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json
+++ b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json
@@ -26,6 +26,7 @@
                             "urn:li:chart:(superset,11)"
                         ],
                         "datasets": [],
+                        "dashboards": [],
                         "lastModified": {
                             "created": {
                                 "time": 0,
@@ -44,7 +45,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00",
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -92,7 +93,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00",
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -140,7 +141,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00",
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -188,47 +189,413 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00",
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
 },
 {
     "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
-            "urn": "urn:li:chart:(superset,13)",
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
             "aspects": [
                 {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": []
                     }
                 },
                 {
-                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
-                        "customProperties": {
-                            "Metrics": "",
-                            "Filters": "",
-                            "Dimensions": ""
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
+                        "description": "",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
                         },
-                        "title": "test_chart_title_4",
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
                         "description": "",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
                         "lastModified": {
-                            "created": {
-                                "time": 0,
-                                "actor": "urn:li:corpuser:unknown"
-                            },
-                            "lastModified": {
-                                "time": 1586847600000,
-                                "actor": "urn:li:corpuser:test_username_2"
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
                             }
                         },
-                        "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_13",
-                        "inputs": [
-                            {
-                                "string": "urn:li:dataset:(urn:li:dataPlatform:external,test_database_name.test_schema_name.test_table_name,PROD)"
+                        "fields": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
+                        "description": "",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
                             }
-                        ],
-                        "type": "HISTOGRAM"
+                        },
+                        "fields": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
+                        "description": "",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
+                        "description": "",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
+                        "description": "",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
+                        "description": "",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
+                        "description": "",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "Test Table 2",
+                        "platform": "urn:li:dataPlatform:superset",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "externalUrl": "mock://mock-domain.superset.com",
+                        "name": "Test Table 2",
+                        "description": "",
+                        "tags": []
                     }
                 }
             ]
@@ -236,7 +603,41 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00",
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema1.Test Table 1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": true
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -253,7 +654,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00",
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(superset,13)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": true
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-83v7ts",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
diff --git a/metadata-ingestion/tests/integration/superset/test_superset.py b/metadata-ingestion/tests/integration/superset/test_superset.py
index b3b5982016146..e8251e54a1f85 100644
--- a/metadata-ingestion/tests/integration/superset/test_superset.py
+++ b/metadata-ingestion/tests/integration/superset/test_superset.py
@@ -133,6 +133,222 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
                 ],
             },
         },
+        "mock://mock-domain.superset.com/api/v1/dataset/": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "count": 215,
+                "description_columns": {},
+                "ids": [1, 2, 3],
+                "result": [
+                    {
+                        "changed_by": {
+                            "first_name": "Test",
+                            "id": 1,
+                            "last_name": "User1",
+                            "username": "test_username_1",
+                        },
+                        "changed_by_name": "test_username_1",
+                        "changed_on_delta_humanized": "10 months ago",
+                        "changed_on_utc": "2024-01-05T21:10:15.650819+0000",
+                        "database": {"database_name": "test_database1", "id": 1},
+                        "datasource_type": "table",
+                        "default_endpoint": None,
+                        "description": None,
+                        "explore_url": "/explore/?datasource_type=table&datasource_id=1",
+                        "extra": None,
+                        "id": 1,
+                        "kind": "virtual",
+                        "owners": [
+                            {
+                                "first_name": "Test",
+                                "id": 1,
+                                "last_name": "Owner1",
+                                "username": "test_username_1",
+                            }
+                        ],
+                        "schema": "test_schema1",
+                        "sql": "SELECT * FROM test_table1",
+                        "table_name": "Test Table 1",
+                    },
+                    {
+                        "changed_by": {
+                            "first_name": "Test",
+                            "id": 2,
+                            "last_name": "User2",
+                            "username": "test_username_2",
+                        },
+                        "changed_by_name": "test_username_2",
+                        "changed_on_delta_humanized": "9 months ago",
+                        "changed_on_utc": "2024-02-10T15:30:20.123456+0000",
+                        "database": {"database_name": "test_database2", "id": 2},
+                        "datasource_type": "table",
+                        "default_endpoint": None,
+                        "description": "Sample description for dataset 2",
+                        "explore_url": "/explore/?datasource_type=table&datasource_id=2",
+                        "extra": None,
+                        "id": 2,
+                        "kind": "physical",
+                        "owners": [
+                            {
+                                "first_name": "Test",
+                                "id": 2,
+                                "last_name": "Owner2",
+                                "username": "test_username_2",
+                            }
+                        ],
+                        "schema": "test_schema2",
+                        "sql": "SELECT * FROM test_table2",
+                        "table_name": "Test Table 2",
+                    },
+                ],
+            },
+        },
+        "mock://mock-domain.superset.com/api/v1/dataset/1": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "id": 1,
+                "result": {
+                    "always_filter_main_dttm": False,
+                    "cache_timeout": None,
+                    "changed_by": {"first_name": "Test", "last_name": "User1"},
+                    "changed_on": "2024-01-05T21:10:15.650819+0000",
+                    "changed_on_humanized": "10 months ago",
+                    "created_by": {"first_name": "Test", "last_name": "User1"},
+                    "created_on": "2024-01-05T21:10:15.650819+0000",
+                    "created_on_humanized": "10 months ago",
+                    "currency_formats": {},
+                    "database": {
+                        "backend": "postgresql",
+                        "database_name": "test_database1",
+                        "id": 1,
+                    },
+                    "datasource_name": "Test Table 1",
+                    "datasource_type": "table",
+                    "default_endpoint": None,
+                    "description": None,
+                    "extra": None,
+                    "fetch_values_predicate": None,
+                    "filter_select_enabled": True,
+                    "granularity_sqla": [
+                        ["created_at", "created_at"],
+                        ["updated_at", "updated_at"],
+                    ],
+                    "id": 1,
+                    "is_managed_externally": False,
+                    "is_sqllab_view": False,
+                    "kind": "virtual",
+                    "main_dttm_col": None,
+                    "metrics": [
+                        {
+                            "changed_on": "2024-01-05T21:10:15.650819+0000",
+                            "created_on": "2024-01-05T21:10:15.650819+0000",
+                            "currency": None,
+                            "d3format": None,
+                            "description": None,
+                            "expression": "count(*)",
+                            "extra": None,
+                            "id": 1,
+                            "metric_name": "count",
+                            "metric_type": None,
+                            "rendered_expression": "count(*)",
+                            "verbose_name": None,
+                            "warning_text": None,
+                        }
+                    ],
+                    "name": "Test Table 1",
+                    "normalize_columns": True,
+                    "offset": 0,
+                    "owners": [{"first_name": "Test", "id": 1, "last_name": "Owner1"}],
+                    "rendered_sql": "SELECT * FROM test_table1",
+                    "schema": "test_schema1",
+                    "select_star": "SELECT * FROM test_schema1.test_table1 LIMIT 100",
+                    "sql": "SELECT * FROM test_table1",
+                    "table_name": "Test Table 1",
+                    "uid": "1__table",
+                    "url": "/tablemodelview/edit/1",
+                    "verbose_map": {
+                        "__timestamp": "Time",
+                        "id": "ID",
+                        "name": "Name",
+                        "created_at": "Created At",
+                        "updated_at": "Updated At",
+                    },
+                },
+            },
+        },
+        "mock://mock-domain.superset.com/api/v1/dataset/2": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "id": 2,
+                "result": {
+                    "always_filter_main_dttm": False,
+                    "cache_timeout": None,
+                    "changed_by": {"first_name": "Test", "last_name": "User2"},
+                    "changed_on": "2024-02-10T15:30:20.123456+0000",
+                    "changed_on_humanized": "9 months ago",
+                    "created_by": {"first_name": "Test", "last_name": "User2"},
+                    "created_on": "2024-02-10T15:30:20.123456+0000",
+                    "created_on_humanized": "9 months ago",
+                    "currency_formats": {},
+                    "database": {
+                        "backend": "postgresql",
+                        "database_name": "test_database1",
+                        "id": 1,
+                    },
+                    "datasource_name": "Test Table 2",
+                    "datasource_type": "table",
+                    "default_endpoint": None,
+                    "description": "Sample description for dataset 2",
+                    "extra": None,
+                    "fetch_values_predicate": None,
+                    "filter_select_enabled": True,
+                    "granularity_sqla": [["date_column", "date_column"]],
+                    "id": 2,
+                    "is_managed_externally": False,
+                    "is_sqllab_view": True,
+                    "kind": "virtual",
+                    "main_dttm_col": "date_column",
+                    "metrics": [
+                        {
+                            "changed_on": "2024-02-10T15:30:20.123456+0000",
+                            "created_on": "2024-02-10T15:30:20.123456+0000",
+                            "currency": None,
+                            "d3format": None,
+                            "description": None,
+                            "expression": "sum(value)",
+                            "extra": None,
+                            "id": 2,
+                            "metric_name": "total_value",
+                            "metric_type": None,
+                            "rendered_expression": "sum(value)",
+                            "verbose_name": "Total Value",
+                            "warning_text": None,
+                        }
+                    ],
+                    "name": "Test Table 2",
+                    "normalize_columns": True,
+                    "offset": 0,
+                    "owners": [{"first_name": "Test", "id": 2, "last_name": "Owner2"}],
+                    "rendered_sql": "SELECT * FROM test_table2",
+                    "schema": "test_schema2",
+                    "select_star": "SELECT * FROM test_schema2.test_table2 LIMIT 100",
+                    "sql": "SELECT * FROM test_table2",
+                    "table_name": "Test Table 2",
+                    "uid": "2__table",
+                    "url": "/tablemodelview/edit/2",
+                    "verbose_map": {
+                        "__timestamp": "Time",
+                        "id": "ID",
+                        "name": "Name",
+                        "value": "Value",
+                        "date_column": "Date",
+                    },
+                },
+            },
+        },
         "mock://mock-domain.superset.com/api/v1/dataset/20": {
             "method": "GET",
             "status_code": 200,
@@ -147,6 +363,19 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
                 },
             },
         },
+        "mock://mock-domain.superset.com/api/v1/database/1": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "id": 1,
+                "result": {
+                    "configuration_method": "sqlalchemy_form",
+                    "database_name": "test_database1",
+                    "id": 1,
+                    "sqlalchemy_uri": "postgresql://user:password@host:port/test_database1",
+                },
+            },
+        },
         "mock://mock-domain.superset.com/api/v1/database/30": {
             "method": "GET",
             "status_code": 200,
@@ -225,6 +454,8 @@ def test_superset_stateful_ingest(
                 "username": "test_username",
                 "password": "test_password",
                 "provider": "db",
+                # Enable dataset ingestion
+                "ingest_datasets": True,
                 # enable stateful ingestion
                 "stateful_ingestion": {
                     "enabled": True,
@@ -244,7 +475,7 @@ def test_superset_stateful_ingest(
         "pipeline_name": "test_pipeline",
     }
 
-    dashboard_endpoint_override = {
+    asset_override = {
         "mock://mock-domain.superset.com/api/v1/dashboard/": {
             "method": "GET",
             "status_code": 200,
@@ -276,6 +507,92 @@ def test_superset_stateful_ingest(
                 ],
             },
         },
+        "mock://mock-domain.superset.com/api/v1/chart/": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "count": 3,
+                "result": [
+                    {
+                        "id": "10",
+                        "changed_by": {
+                            "username": "test_username_1",
+                        },
+                        "changed_on_utc": "2020-04-14T07:00:00.000000+0000",
+                        "slice_name": "test_chart_title_1",
+                        "viz_type": "box_plot",
+                        "url": "/explore/test_chart_url_10",
+                        "datasource_id": "20",
+                        "params": '{"metrics": [], "adhoc_filters": []}',
+                    },
+                    {
+                        "id": "11",
+                        "changed_by": {
+                            "username": "test_username_1",
+                        },
+                        "changed_on_utc": "2020-04-14T07:00:00.000000+0000",
+                        "slice_name": "test_chart_title_2",
+                        "viz_type": "pie",
+                        "url": "/explore/test_chart_url_11",
+                        "datasource_id": "20",
+                        "params": '{"metrics": [], "adhoc_filters": []}',
+                    },
+                    {
+                        "id": "12",
+                        "changed_by": {
+                            "username": "test_username_2",
+                        },
+                        "changed_on_utc": "2020-04-14T07:00:00.000000+0000",
+                        "slice_name": "test_chart_title_3",
+                        "viz_type": "treemap",
+                        "url": "/explore/test_chart_url_12",
+                        "datasource_id": "20",
+                        "params": '{"metrics": [], "adhoc_filters": []}',
+                    },
+                ],
+            },
+        },
+        "mock://mock-domain.superset.com/api/v1/dataset/": {
+            "method": "GET",
+            "status_code": 200,
+            "json": {
+                "count": 214,
+                "description_columns": {},
+                "ids": [1, 2],
+                "result": [
+                    {
+                        "changed_by": {
+                            "first_name": "Test",
+                            "id": 2,
+                            "last_name": "User2",
+                            "username": "test_username_2",
+                        },
+                        "changed_by_name": "test_username_2",
+                        "changed_on_delta_humanized": "9 months ago",
+                        "changed_on_utc": "2024-02-10T15:30:20.123456+0000",
+                        "database": {"database_name": "test_database1", "id": 1},
+                        "datasource_type": "table",
+                        "default_endpoint": None,
+                        "description": "Sample description for dataset 2",
+                        "explore_url": "/explore/?datasource_type=table&datasource_id=2",
+                        "extra": None,
+                        "id": 2,
+                        "kind": "physical",
+                        "owners": [
+                            {
+                                "first_name": "Test",
+                                "id": 2,
+                                "last_name": "Owner2",
+                                "username": "test_username_2",
+                            }
+                        ],
+                        "schema": "test_schema2",
+                        "sql": "SELECT * FROM test_table2",
+                        "table_name": "Test Table 2",
+                    },
+                ],
+            },
+        },
     }
 
     with patch(
@@ -292,10 +609,8 @@ def test_superset_stateful_ingest(
         assert checkpoint1
         assert checkpoint1.state
 
-        # Remove one dashboard from the superset config.
-        register_mock_api(
-            request_mock=requests_mock, override_data=dashboard_endpoint_override
-        )
+        # Remove one dashboard, chart, dataset from the superset config.
+        register_mock_api(request_mock=requests_mock, override_data=asset_override)
 
         # Capture MCEs of second run to validate Status(removed=true)
         deleted_mces_path = f"{tmp_path}/superset_deleted_mces.json"
@@ -313,15 +628,27 @@ def test_superset_stateful_ingest(
         # part of the second state
         state1 = checkpoint1.state
         state2 = checkpoint2.state
-        difference_urns = list(
+        dashboard_difference_urns = list(
             state1.get_urns_not_in(type="dashboard", other_checkpoint_state=state2)
         )
+        chart_difference_urns = list(
+            state1.get_urns_not_in(type="chart", other_checkpoint_state=state2)
+        )
+        dataset_difference_urns = list(
+            state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2)
+        )
 
-        assert len(difference_urns) == 1
+        assert len(dashboard_difference_urns) == 1
+        assert len(chart_difference_urns) == 1
+        assert len(dataset_difference_urns) == 1
 
         urn1 = "urn:li:dashboard:(superset,2)"
+        urn2 = "urn:li:chart:(superset,13)"
+        urn3 = "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema1.Test Table 1,PROD)"
 
-        assert urn1 in difference_urns
+        assert urn1 in dashboard_difference_urns
+        assert urn2 in chart_difference_urns
+        assert urn3 in dataset_difference_urns
 
         # Validate that all providers have committed successfully.
         validate_all_providers_have_committed_successfully(
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json
index 855f872838052..5901f18bd6643 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json
@@ -870,8 +870,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -939,6 +939,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1221,6 +1226,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1789,6 +1799,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2409,6 +2424,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2925,6 +2945,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3077,6 +3102,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3604,6 +3634,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4112,6 +4147,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4588,6 +4628,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5093,6 +5138,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5485,6 +5535,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5851,6 +5906,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6246,6 +6306,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6693,6 +6758,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7111,6 +7181,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7451,6 +7526,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7817,6 +7897,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8079,6 +8164,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8526,6 +8616,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8869,6 +8964,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9238,6 +9338,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9630,6 +9735,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10077,6 +10187,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10227,8 +10342,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10277,6 +10392,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10384,6 +10504,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10489,6 +10614,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10608,6 +10738,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11038,6 +11173,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -12896,7 +13036,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
                     "type": "TRANSFORMED"
                 },
                 {
@@ -12904,7 +13044,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
                     "type": "TRANSFORMED"
                 }
             ],
@@ -12995,6 +13135,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13336,6 +13481,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14251,6 +14401,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15212,6 +15367,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22084,6 +22244,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26204,6 +26369,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31558,6 +31728,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31902,6 +32077,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32671,6 +32851,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json
index 3fd5ed1ac9dee..f5914d5b2e2d1 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json
@@ -1129,8 +1129,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -1198,6 +1198,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1480,6 +1485,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2048,6 +2058,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2668,6 +2683,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3184,6 +3204,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3336,6 +3361,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3863,6 +3893,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4371,6 +4406,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4847,6 +4887,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5352,6 +5397,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5744,6 +5794,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6110,6 +6165,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6505,6 +6565,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6952,6 +7017,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7370,6 +7440,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7710,6 +7785,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8076,6 +8156,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8338,6 +8423,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8785,6 +8875,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9128,6 +9223,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9497,6 +9597,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9889,6 +9994,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10336,6 +10446,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10486,8 +10601,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10536,6 +10651,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10643,6 +10763,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10748,6 +10873,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10867,6 +10997,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11297,6 +11432,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13155,7 +13295,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
                     "type": "TRANSFORMED"
                 },
                 {
@@ -13163,7 +13303,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
                     "type": "TRANSFORMED"
                 }
             ],
@@ -13254,6 +13394,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13595,6 +13740,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14510,6 +14660,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15471,6 +15626,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22343,6 +22503,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26463,6 +26628,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31817,6 +31987,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32161,6 +32336,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32930,6 +33110,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_hidden_asset_tags_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_hidden_asset_tags_mces_golden.json
new file mode 100644
index 0000000000000..78c72f9d2a871
--- /dev/null
+++ b/metadata-ingestion/tests/integration/tableau/tableau_hidden_asset_tags_mces_golden.json
@@ -0,0 +1,44892 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01"
+            },
+            "name": "default"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c"
+            },
+            "name": "Project 2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d"
+            },
+            "name": "Samples"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "79d02655-88e5-45a6-9f9b-eeaf5fe54903"
+            },
+            "name": "DenyProject"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995",
+            "name": "Email Performance by Campaign",
+            "description": "Description for Email Performance by Campaign"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619",
+            "name": "Dvdrental Workbook",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": [
+                {
+                    "tag": "urn:li:tag:TagSheet3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605",
+            "name": "Executive Dashboard",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904",
+            "name": "Workbook published ds",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "ee012e36-d916-4c21-94ab-f0d66736af4e"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904",
+            "name": "Deny Pattern WorkBook",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+                    "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "chartUsageStatistics",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1638860400000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "viewsCount": 5
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {
+                            "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783"
+                        },
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent",
+                        "title": "Timeline - Sent",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List",
+                        "title": "Campaign List",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Rate",
+                        "nullable": false,
+                        "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary",
+                        "title": "Summary",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Rate",
+                        "nullable": false,
+                        "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)",
+                    "schemaField": {
+                        "fieldPath": "Measure Names",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)",
+                    "schemaField": {
+                        "fieldPath": "Measure Values",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign",
+                        "title": "Mobile - Sent by Campaign",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1",
+                        "title": "Sheet 1",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639772911000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)",
+                    "schemaField": {
+                        "fieldPath": "Custom SQL Query",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.ArrayType": {}
+                            }
+                        },
+                        "nativeDataType": "TABLE",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2",
+                        "title": "Sheet 2",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773415000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)",
+                    "schemaField": {
+                        "fieldPath": "Custom SQL Query",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.ArrayType": {}
+                            }
+                        },
+                        "nativeDataType": "TABLE",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)",
+                    "schemaField": {
+                        "fieldPath": "First Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)",
+                    "schemaField": {
+                        "fieldPath": "Last Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)",
+                    "schemaField": {
+                        "fieldPath": "amount",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)",
+                    "schemaField": {
+                        "fieldPath": "customer_id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)",
+                    "schemaField": {
+                        "fieldPath": "payment_date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:YEAR"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)",
+                    "schemaField": {
+                        "fieldPath": "rental_id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3",
+                        "title": "Sheet 3",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640375456000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:TagSheet3"
+                            },
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)",
+                    "schemaField": {
+                        "fieldPath": "Category",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)",
+                    "schemaField": {
+                        "fieldPath": "Customer Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)",
+                    "schemaField": {
+                        "fieldPath": "Segment",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests",
+                        "title": "Opened Requests",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)",
+                    "schemaField": {
+                        "fieldPath": "Total # Request",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Requests",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change",
+                        "title": "Top 10 Items by Requests and YoY Change",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems",
+                        "title": "Opened Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)",
+                    "schemaField": {
+                        "fieldPath": "Total # Problems",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Problems",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue",
+                        "title": "Overdue",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems",
+                        "title": "High and Critical Priority Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)",
+                    "schemaField": {
+                        "fieldPath": "% of critical and high priority",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change",
+                        "title": "Total Incidents by Category and YoY Change",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors",
+                        "title": "Known Errors",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)",
+                    "schemaField": {
+                        "fieldPath": "% of known error",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)",
+                    "schemaField": {
+                        "fieldPath": "Known error",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Problems with Related Incidents",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Related Incidents",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests",
+                        "title": "Overdue Requests",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident",
+                        "title": "AVG Time to Solve an Incident",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)",
+                    "schemaField": {
+                        "fieldPath": "Time to Close an Incident (seconds)",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)",
+                    "schemaField": {
+                        "fieldPath": "Time to Close an Incident",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F",
+                        "title": "Made SLA?",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)",
+                    "schemaField": {
+                        "fieldPath": "% made SLA",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)",
+                    "schemaField": {
+                        "fieldPath": "Made SLA",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority",
+                        "title": "Tooltip - Incident Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems",
+                        "title": "Overdue Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority",
+                        "title": "Tooltip - Problem Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority",
+                        "title": "Tooltip - Request Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems",
+                        "title": "Age of Active Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)",
+                    "schemaField": {
+                        "fieldPath": "Age of Problems",
+                        "nullable": false,
+                        "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)",
+                    "schemaField": {
+                        "fieldPath": "Time Span Breakdown",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents",
+                        "title": "Opened Incidents",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)",
+                    "schemaField": {
+                        "fieldPath": "Opened Month Tooltip",
+                        "nullable": false,
+                        "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Incidents",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1",
+                        "title": "published sheet ds",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1641951867000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642658093000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)",
+                    "schemaField": {
+                        "fieldPath": "amount",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)",
+                    "schemaField": {
+                        "fieldPath": "customer_first_name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)",
+                    "schemaField": {
+                        "fieldPath": "staff_last_name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardUsageStatistics",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1638860400000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "viewsCount": 3
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {
+                            "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66"
+                        },
+                        "title": "Email Performance by Campaign",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+                            "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+                            "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+                            "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)"
+                        ],
+                        "datasets": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "dvd Rental Dashboard",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)"
+                        ],
+                        "datasets": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773866000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "Story 1",
+                        "description": "",
+                        "charts": [],
+                        "datasets": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773866000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "Executive Dashboard",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+                            "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+                            "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+                            "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+                            "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+                            "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+                            "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+                            "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+                            "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+                            "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+                            "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+                            "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+                            "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)"
+                        ],
+                        "datasets": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:hidden"
+                            },
+                            {
+                                "tag": "urn:li:tag:private"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-02-09T00:05:25Z",
+                            "extractLastUpdateTime": "2018-02-09T00:05:25Z"
+                        },
+                        "name": "Marketo",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Delivery Rate",
+                                "nullable": false,
+                                "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Program ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated At",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workspace Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User Agent (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Predictive",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Clickthrough Rate",
+                                "nullable": false,
+                                "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Open Rate",
+                                "nullable": false,
+                                "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sent Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivered Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Mobile Device",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created At",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Clickthrough Emails",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Mobile Device (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Click-to-Open",
+                                "nullable": false,
+                                "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Non Clicked Emails",
+                                "nullable": false,
+                                "description": "formula: [Opened Email]-[Clickthrough Emails]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Non Clickthrough Email",
+                                "nullable": false,
+                                "description": "formula: [Delivered Email]-[Clickthrough Emails]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Program Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Non Opened Email",
+                                "nullable": false,
+                                "description": "formula: [Delivered Email]-[Opened Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User Agent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Bounceback",
+                                "nullable": false,
+                                "description": "formula: [Sent Email] - [Delivered Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Customer Payment Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Custom SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rental_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "actor+ (dvdrental)",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Address2",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Update",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "District",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "country, city",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Phone",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Update (Address)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Address Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Actor Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "actor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Superstore Datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Region (People)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "People",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID (Returns)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Orders",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit Ratio",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returns",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Regional Manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit (bin)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Top Customers by Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T19:35:39Z",
+                            "extractLastUpdateTime": "2018-01-18T19:35:39Z"
+                        },
+                        "name": "Requests",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Closed by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mobile Picture",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% made SLA",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Meta",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total # Request",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requested for date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ordered item link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible elsewhere",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price Frequency (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Update name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Special instructions",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Execution Plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible on Bundles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No search",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "List Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer update",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Guide",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Package",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Estimated Delivery",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Billable",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price Frequency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fulfillment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Backordered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ignore price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Replace on upgrade",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Context",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requested for",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Billable (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mobile Picture Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Protection policy",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Omit price in cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Catalogs",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Entitlement script",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Published version",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Preview link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Stage (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible on Guides",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Requested Item) 1",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery time",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Stage",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Request",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolve Time",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Icon",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Display name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No proceed checkout",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Use cart layout",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No order now",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Application",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Template",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Catalog",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Image",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workflow",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Class",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created from item design",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Roles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Hide price (mobile listings)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Availability",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Vendor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Picture",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan script",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Requests",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Start closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Request state",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T20:21:33Z",
+                            "extractLastUpdateTime": "2018-01-18T20:21:33Z"
+                        },
+                        "name": "Problems",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "SLA due (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Default assignee",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workaround",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time Span Breakdown",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Change request",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost center",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Roles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total # Problems",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Exclude manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lucha",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problems with Related Incidents",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Related Incidents",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Source",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of known error",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problem state",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Known error",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group email",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age of Problems",
+                                "nullable": false,
+                                "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of critical and high priority",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Include members",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Problems",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T20:13:08Z",
+                            "extractLastUpdateTime": "2018-01-18T20:13:08Z"
+                        },
+                        "name": "Incidents",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Attributes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close code (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Status",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Notify (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requires verification",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Maintenance schedule",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Warranty expiration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "GL account",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First discovered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Asset",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Skip sync",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "DNS Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Caller (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Department",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolved by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Managed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "PO number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business resolve time (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Child Incidents (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "IP Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Asset tag",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due in",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Checked out",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fully qualified domain name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Installed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Purchased",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lease contract",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Vendor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent Incident (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost currency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Subcategory (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Monitor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Serial number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time to Close an Incident (seconds)",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Owned by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Invoice number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Start date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ordered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order received",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discovery source",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Incidents",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Class",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Operational status",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolve time (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reopen count (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Most recent discovery",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fault count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Caused by Change (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "MAC Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Schedule",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Supported by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Support group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Justification",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Change Request (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Incident state (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Month Tooltip",
+                                "nullable": false,
+                                "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problem (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Checked in",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Severity (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost center",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Subcategory (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Can Print",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolved (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time to Close an Incident",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "test publish datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Custom SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-02-09T00:05:25Z",
+                            "extractLastUpdateTime": "2018-02-09T00:05:25Z"
+                        },
+                        "name": "New DataSource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Program ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:tag on published datasource"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "test publish datasource",
+                        "description": "description for test publish datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Published SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Published Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)"
+                    ],
+                    "transformOperation": "HierarchyField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)"
+                    ],
+                    "transformOperation": "GroupField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)"
+                    ],
+                    "transformOperation": "HierarchyField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)"
+                    ],
+                    "transformOperation": "BinField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)"
+                    ],
+                    "transformOperation": "SetField",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Superstore Datasource",
+                        "description": "Description for Superstore dataset",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Top Customers by Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:SETFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returns",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit Ratio",
+                                "nullable": false,
+                                "description": "formula: SUM([Profit])/SUM([Sales])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Orders",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit (bin)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:BINFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID (Returns)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Person",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "People",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:GROUPFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region (People)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Published Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+                    "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "NUMERIC",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rental_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n    ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "NUMERIC",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n    ON p.customer_id = c.customer_id\nINNER JOIN staff s \n    ON p.staff_id = s.staff_id\nORDER BY payment_date",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "seller_city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/SubProject1/AbcJoinWorkbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "SubProject1"
+                },
+                {
+                    "id": "AbcJoinWorkbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Mobile_Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User_Agent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Mobile_Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User_Agent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "programName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "programId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "createdAt",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "workspaceName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "updatedAt",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address2",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I2",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "district",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Dvdrental Workbook"
+                },
+                {
+                    "id": "actor+ (dvdrental)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "last_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "actor_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Dvdrental Workbook"
+                },
+                {
+                    "id": "actor+ (dvdrental)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Person",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems",
+                            "/prod/tableau/default/Executive Dashboard/Requests",
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "requested_for",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "requested_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "request_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "stage",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "special_instructions",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "configuration_item",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_catalog",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "stage",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "estimated_delivery",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "context",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "billable",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_frequency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cat_item",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order_guide",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "backordered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "request",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "sc_catalogs",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_picture_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "workflow",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_customer_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_standalone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_scope",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "template",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_proceed_checkout",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "billable",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "meta",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ordered_item_link",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_ic_version",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "image",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_policy",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "roles",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "picture",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "list_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_order_now",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vendor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_ic_item_staging",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "entitlement_script",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "icon",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ignore_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "start_closed",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_package",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_replace_on_upgrade",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_FLOAT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "use_sc_layout",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "availability",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "custom_cart",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_picture",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_cart",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_hide_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_time",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_search",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_frequency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan_script",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_bundle",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_update_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_guide",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "preview",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "omit_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "u_u",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "source",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "exclude_manager",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_center",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_lucha",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_lu2",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "roles",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "default_assignee",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "manager",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "include_members",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "related_incidents",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rfc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "problem_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_around",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "known_error",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "severity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent_incident",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "subcategory",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "caller_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "resolved_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "resolved_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "child_incidents",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "incident_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "notify",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "problem_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "caused_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reopen_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rfc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Incidents"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "first_discovered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "operational_status",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_discovered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_cc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "checked_in",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "attributes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "serial_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vendor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ip_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "support_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "asset",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "supported_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "invoice_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "managed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fault_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_in",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "justification",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "lease_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "monitor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_center",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "subcategory",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "can_print",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "start_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "discovery_source",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "schedule",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fqdn",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warranty_expiration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "owned_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "asset_tag",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "manufacturer",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "purchase_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "department",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "checked_out",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "unverified",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "skip_sync",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "po_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "gl_account",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "maintenance_schedule",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "install_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "dns_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mac_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "change_control",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "install_status",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Incidents"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Customer Payment Query",
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Customer Payment Query"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Customer Payment Query",
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Customer Payment Query"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "test publish datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:ATTRIBUTE",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "ATTRIBUTE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:BINFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "BINFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:CALCULATEDFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "CALCULATEDFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:COLUMNFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "COLUMNFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:COUNT",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "COUNT"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:DATASOURCEFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "DATASOURCEFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:DIMENSION",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "DIMENSION"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:GROUPFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "GROUPFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:HIERARCHYFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "HIERARCHYFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:MEASURE",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "MEASURE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:SETFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "SETFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:SUM",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "SUM"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:TagSheet3",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "TagSheet3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:YEAR",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "YEAR"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:hidden",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "hidden"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:private",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "private"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:tag on published datasource",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "tag on published datasource"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_hidden_asset_tags_ingest"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_ingest_tags_disabled_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_ingest_tags_disabled_mces_golden.json
new file mode 100644
index 0000000000000..b0946fd053f75
--- /dev/null
+++ b/metadata-ingestion/tests/integration/tableau/tableau_ingest_tags_disabled_mces_golden.json
@@ -0,0 +1,32956 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01"
+            },
+            "name": "default"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c"
+            },
+            "name": "Project 2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d"
+            },
+            "name": "Samples"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995",
+            "name": "Email Performance by Campaign",
+            "description": "Description for Email Performance by Campaign"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619",
+            "name": "Dvdrental Workbook",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605",
+            "name": "Executive Dashboard",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904",
+            "name": "Workbook published ds",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "chartUsageStatistics",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1638860400000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "viewsCount": 5
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {
+                            "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783"
+                        },
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent",
+                        "title": "Timeline - Sent",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List",
+                        "title": "Campaign List",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Rate",
+                        "nullable": false,
+                        "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary",
+                        "title": "Summary",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Rate",
+                        "nullable": false,
+                        "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)",
+                    "schemaField": {
+                        "fieldPath": "Measure Names",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)",
+                    "schemaField": {
+                        "fieldPath": "Measure Values",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign",
+                        "title": "Mobile - Sent by Campaign",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1",
+                        "title": "Sheet 1",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639772911000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)",
+                    "schemaField": {
+                        "fieldPath": "Custom SQL Query",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.ArrayType": {}
+                            }
+                        },
+                        "nativeDataType": "TABLE",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2",
+                        "title": "Sheet 2",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773415000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)",
+                    "schemaField": {
+                        "fieldPath": "Custom SQL Query",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.ArrayType": {}
+                            }
+                        },
+                        "nativeDataType": "TABLE",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)",
+                    "schemaField": {
+                        "fieldPath": "First Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)",
+                    "schemaField": {
+                        "fieldPath": "Last Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)",
+                    "schemaField": {
+                        "fieldPath": "amount",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)",
+                    "schemaField": {
+                        "fieldPath": "customer_id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)",
+                    "schemaField": {
+                        "fieldPath": "payment_date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)",
+                    "schemaField": {
+                        "fieldPath": "rental_id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3",
+                        "title": "Sheet 3",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640375456000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)",
+                    "schemaField": {
+                        "fieldPath": "Category",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)",
+                    "schemaField": {
+                        "fieldPath": "Customer Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)",
+                    "schemaField": {
+                        "fieldPath": "Segment",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests",
+                        "title": "Opened Requests",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)",
+                    "schemaField": {
+                        "fieldPath": "Total # Request",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Requests",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change",
+                        "title": "Top 10 Items by Requests and YoY Change",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems",
+                        "title": "Opened Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)",
+                    "schemaField": {
+                        "fieldPath": "Total # Problems",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Problems",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue",
+                        "title": "Overdue",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems",
+                        "title": "High and Critical Priority Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)",
+                    "schemaField": {
+                        "fieldPath": "% of critical and high priority",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change",
+                        "title": "Total Incidents by Category and YoY Change",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors",
+                        "title": "Known Errors",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)",
+                    "schemaField": {
+                        "fieldPath": "% of known error",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)",
+                    "schemaField": {
+                        "fieldPath": "Known error",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Problems with Related Incidents",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Related Incidents",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests",
+                        "title": "Overdue Requests",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident",
+                        "title": "AVG Time to Solve an Incident",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)",
+                    "schemaField": {
+                        "fieldPath": "Time to Close an Incident (seconds)",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)",
+                    "schemaField": {
+                        "fieldPath": "Time to Close an Incident",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F",
+                        "title": "Made SLA?",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)",
+                    "schemaField": {
+                        "fieldPath": "% made SLA",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)",
+                    "schemaField": {
+                        "fieldPath": "Made SLA",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority",
+                        "title": "Tooltip - Incident Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems",
+                        "title": "Overdue Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority",
+                        "title": "Tooltip - Problem Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority",
+                        "title": "Tooltip - Request Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems",
+                        "title": "Age of Active Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)",
+                    "schemaField": {
+                        "fieldPath": "Age of Problems",
+                        "nullable": false,
+                        "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)",
+                    "schemaField": {
+                        "fieldPath": "Time Span Breakdown",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents",
+                        "title": "Opened Incidents",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)",
+                    "schemaField": {
+                        "fieldPath": "Opened Month Tooltip",
+                        "nullable": false,
+                        "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Incidents",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1",
+                        "title": "published sheet ds",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1641951867000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642658093000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)",
+                    "schemaField": {
+                        "fieldPath": "amount",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)",
+                    "schemaField": {
+                        "fieldPath": "customer_first_name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)",
+                    "schemaField": {
+                        "fieldPath": "staff_last_name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardUsageStatistics",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1638860400000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "viewsCount": 3
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {
+                            "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66"
+                        },
+                        "title": "Email Performance by Campaign",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+                            "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+                            "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+                            "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)"
+                        ],
+                        "datasets": [],
+                        "dashboards": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "dvd Rental Dashboard",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)"
+                        ],
+                        "datasets": [],
+                        "dashboards": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773866000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "Story 1",
+                        "description": "",
+                        "charts": [],
+                        "datasets": [],
+                        "dashboards": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773866000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "Executive Dashboard",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+                            "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+                            "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+                            "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+                            "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+                            "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+                            "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+                            "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+                            "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+                            "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+                            "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+                            "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+                            "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)"
+                        ],
+                        "datasets": [],
+                        "dashboards": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-02-09T00:05:25Z",
+                            "extractLastUpdateTime": "2018-02-09T00:05:25Z"
+                        },
+                        "name": "Marketo",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Delivery Rate",
+                                "nullable": false,
+                                "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Program ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated At",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workspace Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User Agent (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Predictive",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Clickthrough Rate",
+                                "nullable": false,
+                                "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Open Rate",
+                                "nullable": false,
+                                "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sent Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivered Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Mobile Device",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created At",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Clickthrough Emails",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Mobile Device (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Click-to-Open",
+                                "nullable": false,
+                                "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Non Clicked Emails",
+                                "nullable": false,
+                                "description": "formula: [Opened Email]-[Clickthrough Emails]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Non Clickthrough Email",
+                                "nullable": false,
+                                "description": "formula: [Delivered Email]-[Clickthrough Emails]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Program Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Non Opened Email",
+                                "nullable": false,
+                                "description": "formula: [Delivered Email]-[Opened Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User Agent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Bounceback",
+                                "nullable": false,
+                                "description": "formula: [Sent Email] - [Delivered Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Customer Payment Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Custom SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rental_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "actor+ (dvdrental)",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Address2",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Update",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "District",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "country, city",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Phone",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Update (Address)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Address Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Actor Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "actor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Superstore Datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Region (People)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "People",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID (Returns)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Orders",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit Ratio",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returns",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Regional Manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit (bin)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Top Customers by Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T19:35:39Z",
+                            "extractLastUpdateTime": "2018-01-18T19:35:39Z"
+                        },
+                        "name": "Requests",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Closed by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mobile Picture",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% made SLA",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Meta",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total # Request",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requested for date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ordered item link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible elsewhere",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price Frequency (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Update name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Special instructions",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Execution Plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible on Bundles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No search",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "List Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer update",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Guide",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Package",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Estimated Delivery",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Billable",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price Frequency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fulfillment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Backordered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ignore price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Replace on upgrade",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Context",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requested for",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Billable (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mobile Picture Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Protection policy",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Omit price in cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Catalogs",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Entitlement script",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Published version",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Preview link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Stage (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible on Guides",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Requested Item) 1",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery time",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Stage",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Request",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolve Time",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Icon",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Display name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No proceed checkout",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Use cart layout",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No order now",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Application",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Template",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Catalog",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Image",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workflow",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Class",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created from item design",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Roles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Hide price (mobile listings)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Availability",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Vendor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Picture",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan script",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Requests",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Start closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Request state",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T20:21:33Z",
+                            "extractLastUpdateTime": "2018-01-18T20:21:33Z"
+                        },
+                        "name": "Problems",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "SLA due (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Default assignee",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workaround",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time Span Breakdown",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Change request",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost center",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Roles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total # Problems",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Exclude manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lucha",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problems with Related Incidents",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Related Incidents",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Source",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of known error",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problem state",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Known error",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group email",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age of Problems",
+                                "nullable": false,
+                                "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of critical and high priority",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Include members",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Problems",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T20:13:08Z",
+                            "extractLastUpdateTime": "2018-01-18T20:13:08Z"
+                        },
+                        "name": "Incidents",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Attributes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close code (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Status",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Notify (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requires verification",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Maintenance schedule",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Warranty expiration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "GL account",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First discovered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Asset",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Skip sync",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "DNS Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Caller (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Department",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolved by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Managed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "PO number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business resolve time (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Child Incidents (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "IP Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Asset tag",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due in",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Checked out",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fully qualified domain name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Installed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Purchased",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lease contract",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Vendor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent Incident (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost currency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Subcategory (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Monitor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Serial number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time to Close an Incident (seconds)",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Owned by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Invoice number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Start date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ordered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order received",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discovery source",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Incidents",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Class",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Operational status",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolve time (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reopen count (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Most recent discovery",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fault count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Caused by Change (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "MAC Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Schedule",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Supported by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Support group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Justification",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Change Request (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Incident state (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Month Tooltip",
+                                "nullable": false,
+                                "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problem (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Checked in",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Severity (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost center",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Subcategory (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Can Print",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolved (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time to Close an Incident",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "test publish datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Custom SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-02-09T00:05:25Z",
+                            "extractLastUpdateTime": "2018-02-09T00:05:25Z"
+                        },
+                        "name": "New DataSource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Program ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "test publish datasource",
+                        "description": "description for test publish datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Published SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Published Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)"
+                    ],
+                    "transformOperation": "HierarchyField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)"
+                    ],
+                    "transformOperation": "GroupField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)"
+                    ],
+                    "transformOperation": "HierarchyField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)"
+                    ],
+                    "transformOperation": "BinField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)"
+                    ],
+                    "transformOperation": "SetField",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Superstore Datasource",
+                        "description": "Description for Superstore dataset",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Top Customers by Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returns",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit Ratio",
+                                "nullable": false,
+                                "description": "formula: SUM([Profit])/SUM([Sales])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Orders",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit (bin)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID (Returns)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Person",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "People",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region (People)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Published Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+                    "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "NUMERIC",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rental_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n    ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "NUMERIC",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n    ON p.customer_id = c.customer_id\nINNER JOIN staff s \n    ON p.staff_id = s.staff_id\nORDER BY payment_date",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "seller_city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/SubProject1/AbcJoinWorkbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "SubProject1"
+                },
+                {
+                    "id": "AbcJoinWorkbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Mobile_Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User_Agent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Mobile_Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User_Agent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "programName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "programId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "createdAt",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "workspaceName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "updatedAt",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address2",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I2",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "district",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Dvdrental Workbook"
+                },
+                {
+                    "id": "actor+ (dvdrental)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "last_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "actor_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Dvdrental Workbook"
+                },
+                {
+                    "id": "actor+ (dvdrental)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Person",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems",
+                            "/prod/tableau/default/Executive Dashboard/Requests",
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "requested_for",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "requested_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "request_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "stage",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "special_instructions",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "configuration_item",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_catalog",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "stage",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "estimated_delivery",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "context",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "billable",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_frequency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cat_item",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order_guide",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "backordered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "request",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "sc_catalogs",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_picture_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "workflow",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_customer_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_standalone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_scope",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "template",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_proceed_checkout",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "billable",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "meta",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ordered_item_link",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_ic_version",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "image",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_policy",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "roles",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "picture",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "list_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_order_now",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vendor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_ic_item_staging",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "entitlement_script",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "icon",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ignore_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "start_closed",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_package",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_replace_on_upgrade",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_FLOAT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "use_sc_layout",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "availability",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "custom_cart",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_picture",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_cart",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_hide_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_time",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_search",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_frequency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan_script",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_bundle",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_update_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_guide",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "preview",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "omit_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "u_u",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "source",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "exclude_manager",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_center",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_lucha",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_lu2",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "roles",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "default_assignee",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "manager",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "include_members",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "related_incidents",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rfc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "problem_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_around",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "known_error",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "severity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent_incident",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "subcategory",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "caller_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "resolved_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "resolved_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "child_incidents",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "incident_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "notify",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "problem_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "caused_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reopen_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rfc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Incidents"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "first_discovered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "operational_status",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_discovered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_cc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "checked_in",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "attributes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "serial_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vendor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ip_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "support_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "asset",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "supported_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "invoice_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "managed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fault_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_in",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "justification",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "lease_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "monitor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_center",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "subcategory",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "can_print",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "start_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "discovery_source",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "schedule",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fqdn",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warranty_expiration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "owned_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "asset_tag",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "manufacturer",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "purchase_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "department",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "checked_out",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "unverified",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "skip_sync",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "po_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "gl_account",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "maintenance_schedule",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "install_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "dns_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mac_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "change_control",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "install_status",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Incidents"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Customer Payment Query",
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Customer Payment Query"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Customer Payment Query",
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Customer Payment Query"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "test publish datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_ingest_tags_disabled"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json
index 29e1b77869aa5..8610f809ac52b 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json
@@ -870,8 +870,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -939,6 +939,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1221,6 +1226,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1789,6 +1799,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2409,6 +2424,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2925,6 +2945,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3077,6 +3102,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3604,6 +3634,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4112,6 +4147,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4588,6 +4628,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5093,6 +5138,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5485,6 +5535,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5851,6 +5906,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6246,6 +6306,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6693,6 +6758,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7111,6 +7181,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7451,6 +7526,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7817,6 +7897,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8079,6 +8164,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8526,6 +8616,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8869,6 +8964,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9238,6 +9338,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9630,6 +9735,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10077,6 +10187,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10227,8 +10342,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10277,6 +10392,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10384,6 +10504,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10489,6 +10614,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10608,6 +10738,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11038,6 +11173,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -12896,7 +13036,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
                     "type": "TRANSFORMED"
                 },
                 {
@@ -12904,7 +13044,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
                     "type": "TRANSFORMED"
                 }
             ],
@@ -12995,6 +13135,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13336,6 +13481,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14251,6 +14401,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15212,6 +15367,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22084,6 +22244,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26204,6 +26369,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31558,6 +31728,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31902,6 +32077,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32671,6 +32851,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json
index 12c665235e6ac..05ec0e76f92d6 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json
@@ -1043,8 +1043,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -1112,6 +1112,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1398,6 +1403,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1970,6 +1980,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2594,6 +2609,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3114,6 +3134,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3270,6 +3295,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3805,6 +3835,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4317,6 +4352,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4797,6 +4837,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5306,6 +5351,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5702,6 +5752,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6072,6 +6127,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6471,6 +6531,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6922,6 +6987,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7344,6 +7414,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7688,6 +7763,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8058,6 +8138,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8324,6 +8409,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8775,6 +8865,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9122,6 +9217,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9495,6 +9595,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9891,6 +9996,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10342,6 +10452,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10496,8 +10611,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10546,6 +10661,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10657,6 +10777,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10766,6 +10891,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10889,6 +11019,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11323,6 +11458,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13284,6 +13424,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13629,6 +13774,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14548,6 +14698,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15513,6 +15668,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22389,6 +22549,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26513,6 +26678,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31871,6 +32041,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32219,6 +32394,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32996,6 +33176,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -44181,8 +44366,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -44250,6 +44435,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -44506,6 +44696,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -45048,6 +45243,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -45642,6 +45842,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -46132,6 +46337,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -46258,6 +46468,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -46733,6 +46948,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -47215,6 +47435,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -47665,6 +47890,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -48144,6 +48374,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -48510,6 +48745,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -48850,6 +49090,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -49219,6 +49464,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -49640,6 +49890,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -50032,6 +50287,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -50346,6 +50606,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -50686,6 +50951,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -50922,6 +51192,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -51343,6 +51618,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -51660,6 +51940,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -52003,6 +52288,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -52369,6 +52659,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -52790,6 +53085,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -52914,8 +53214,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -52964,6 +53264,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/site2/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -53045,6 +53350,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -53124,6 +53434,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -53217,6 +53532,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -53621,6 +53941,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -55552,6 +55877,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -55867,6 +56197,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -56756,6 +57091,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -57691,6 +58031,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -64537,6 +64882,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -68631,6 +68981,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -73959,6 +74314,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -74277,6 +74637,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -74998,6 +75363,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json
index 2aee4abc4d049..dd852671d760c 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json
@@ -1129,8 +1129,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -1198,6 +1198,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1480,6 +1485,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2048,6 +2058,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2668,6 +2683,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3184,6 +3204,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3336,6 +3361,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3863,6 +3893,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4371,6 +4406,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4847,6 +4887,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5352,6 +5397,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5744,6 +5794,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6110,6 +6165,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6505,6 +6565,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6952,6 +7017,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7370,6 +7440,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7710,6 +7785,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8076,6 +8156,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8338,6 +8423,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8785,6 +8875,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9128,6 +9223,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9497,6 +9597,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9889,6 +9994,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10336,6 +10446,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10486,8 +10601,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10536,6 +10651,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10643,6 +10763,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10748,6 +10873,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10867,6 +10997,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11297,6 +11432,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13155,7 +13295,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
                     "type": "TRANSFORMED"
                 },
                 {
@@ -13163,7 +13303,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
                     "type": "TRANSFORMED"
                 }
             ],
@@ -13254,6 +13394,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13595,6 +13740,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14510,6 +14660,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15471,6 +15626,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22343,6 +22503,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26463,6 +26628,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31817,6 +31987,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32161,6 +32336,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32930,6 +33110,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_no_hidden_assets_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_no_hidden_assets_mces_golden.json
new file mode 100644
index 0000000000000..db1d7b4ba25d5
--- /dev/null
+++ b/metadata-ingestion/tests/integration/tableau/tableau_no_hidden_assets_mces_golden.json
@@ -0,0 +1,34728 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01"
+            },
+            "name": "default"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c"
+            },
+            "name": "Project 2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d"
+            },
+            "name": "Samples"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "79d02655-88e5-45a6-9f9b-eeaf5fe54903"
+            },
+            "name": "DenyProject"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995",
+            "name": "Email Performance by Campaign",
+            "description": "Description for Email Performance by Campaign"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619",
+            "name": "Dvdrental Workbook",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": [
+                {
+                    "tag": "urn:li:tag:TagSheet3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605",
+            "name": "Executive Dashboard",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904",
+            "name": "Workbook published ds",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "ee012e36-d916-4c21-94ab-f0d66736af4e"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904",
+            "name": "Deny Pattern WorkBook",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce",
+                    "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "chartUsageStatistics",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1638860400000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "viewsCount": 5
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {
+                            "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783"
+                        },
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent",
+                        "title": "Timeline - Sent",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardUsageStatistics",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1638860400000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "viewsCount": 3
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {
+                            "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66"
+                        },
+                        "title": "Email Performance by Campaign",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+                            "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+                            "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+                            "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)"
+                        ],
+                        "datasets": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-02-09T00:05:25Z",
+                            "extractLastUpdateTime": "2018-02-09T00:05:25Z"
+                        },
+                        "name": "Marketo",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Delivery Rate",
+                                "nullable": false,
+                                "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Program ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated At",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workspace Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User Agent (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Predictive",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Clickthrough Rate",
+                                "nullable": false,
+                                "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Open Rate",
+                                "nullable": false,
+                                "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sent Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivered Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Mobile Device",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created At",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Clickthrough Emails",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Mobile Device (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Click-to-Open",
+                                "nullable": false,
+                                "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Non Clicked Emails",
+                                "nullable": false,
+                                "description": "formula: [Opened Email]-[Clickthrough Emails]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Non Clickthrough Email",
+                                "nullable": false,
+                                "description": "formula: [Delivered Email]-[Clickthrough Emails]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Program Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Non Opened Email",
+                                "nullable": false,
+                                "description": "formula: [Delivered Email]-[Opened Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User Agent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Bounceback",
+                                "nullable": false,
+                                "description": "formula: [Sent Email] - [Delivered Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Customer Payment Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Custom SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rental_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "actor+ (dvdrental)",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Address2",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Update",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "District",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "country, city",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Phone",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Update (Address)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Address Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Actor Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "actor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Superstore Datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Region (People)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "People",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID (Returns)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Orders",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit Ratio",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returns",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Regional Manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit (bin)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Top Customers by Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T19:35:39Z",
+                            "extractLastUpdateTime": "2018-01-18T19:35:39Z"
+                        },
+                        "name": "Requests",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Closed by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mobile Picture",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% made SLA",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Meta",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total # Request",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requested for date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ordered item link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible elsewhere",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price Frequency (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Update name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Special instructions",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Execution Plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible on Bundles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No search",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "List Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer update",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Guide",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Package",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Estimated Delivery",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Billable",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price Frequency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fulfillment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Backordered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ignore price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Replace on upgrade",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Context",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requested for",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Billable (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mobile Picture Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Protection policy",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Omit price in cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Catalogs",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Entitlement script",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Published version",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Preview link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Stage (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible on Guides",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Requested Item) 1",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery time",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Stage",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Request",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolve Time",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Icon",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Display name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No proceed checkout",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Use cart layout",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No order now",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Application",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Template",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Catalog",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Image",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workflow",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Class",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created from item design",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Roles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Hide price (mobile listings)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Availability",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Vendor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Picture",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan script",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Requests",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Start closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Request state",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T20:21:33Z",
+                            "extractLastUpdateTime": "2018-01-18T20:21:33Z"
+                        },
+                        "name": "Problems",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "SLA due (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Default assignee",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workaround",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time Span Breakdown",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Change request",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost center",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Roles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total # Problems",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Exclude manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lucha",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problems with Related Incidents",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Related Incidents",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Source",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of known error",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problem state",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Known error",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group email",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age of Problems",
+                                "nullable": false,
+                                "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of critical and high priority",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Include members",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Problems",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T20:13:08Z",
+                            "extractLastUpdateTime": "2018-01-18T20:13:08Z"
+                        },
+                        "name": "Incidents",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Attributes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close code (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Status",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Notify (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requires verification",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Maintenance schedule",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Warranty expiration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "GL account",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First discovered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Asset",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Skip sync",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "DNS Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Caller (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Department",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolved by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Managed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "PO number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business resolve time (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Child Incidents (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "IP Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Asset tag",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due in",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Checked out",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fully qualified domain name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Installed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Purchased",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lease contract",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Vendor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent Incident (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost currency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Subcategory (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Monitor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Serial number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time to Close an Incident (seconds)",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Owned by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Invoice number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Start date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ordered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order received",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discovery source",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Incidents",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Class",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Operational status",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolve time (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reopen count (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Most recent discovery",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fault count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Caused by Change (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "MAC Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Schedule",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Supported by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Support group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Justification",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Change Request (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Incident state (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Month Tooltip",
+                                "nullable": false,
+                                "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problem (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Checked in",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Severity (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost center",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Subcategory (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Can Print",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolved (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time to Close an Incident",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "test publish datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Custom SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-02-09T00:05:25Z",
+                            "extractLastUpdateTime": "2018-02-09T00:05:25Z"
+                        },
+                        "name": "New DataSource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Program ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:tag on published datasource"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "test publish datasource",
+                        "description": "description for test publish datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Published SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Published Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)"
+                    ],
+                    "transformOperation": "HierarchyField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)"
+                    ],
+                    "transformOperation": "GroupField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)"
+                    ],
+                    "transformOperation": "HierarchyField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)"
+                    ],
+                    "transformOperation": "BinField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)"
+                    ],
+                    "transformOperation": "SetField",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Superstore Datasource",
+                        "description": "Description for Superstore dataset",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Top Customers by Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:SETFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returns",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit Ratio",
+                                "nullable": false,
+                                "description": "formula: SUM([Profit])/SUM([Sales])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Orders",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit (bin)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:BINFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID (Returns)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Person",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "People",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:GROUPFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region (People)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Published Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+                    "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "NUMERIC",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rental_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n    ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "NUMERIC",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n    ON p.customer_id = c.customer_id\nINNER JOIN staff s \n    ON p.staff_id = s.staff_id\nORDER BY payment_date",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "seller_city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/SubProject1/AbcJoinWorkbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "SubProject1"
+                },
+                {
+                    "id": "AbcJoinWorkbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Mobile_Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User_Agent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Mobile_Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User_Agent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "programName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "programId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "createdAt",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "workspaceName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "updatedAt",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address2",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I2",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "district",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Dvdrental Workbook"
+                },
+                {
+                    "id": "actor+ (dvdrental)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "last_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "actor_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Dvdrental Workbook"
+                },
+                {
+                    "id": "actor+ (dvdrental)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Person",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems",
+                            "/prod/tableau/default/Executive Dashboard/Requests",
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "requested_for",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "requested_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "request_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "stage",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "special_instructions",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "configuration_item",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_catalog",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "stage",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "estimated_delivery",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "context",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "billable",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_frequency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cat_item",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order_guide",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "backordered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "request",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "sc_catalogs",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_picture_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "workflow",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_customer_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_standalone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_scope",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "template",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_proceed_checkout",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "billable",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "meta",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ordered_item_link",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_ic_version",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "image",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_policy",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "roles",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "picture",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "list_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_order_now",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vendor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_ic_item_staging",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "entitlement_script",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "icon",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ignore_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "start_closed",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_package",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_replace_on_upgrade",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_FLOAT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "use_sc_layout",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "availability",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "custom_cart",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_picture",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_cart",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_hide_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_time",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_search",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_frequency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan_script",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_bundle",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_update_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_guide",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "preview",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "omit_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "u_u",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "source",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "exclude_manager",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_center",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_lucha",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_lu2",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "roles",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "default_assignee",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "manager",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "include_members",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "related_incidents",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rfc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "problem_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_around",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "known_error",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "severity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent_incident",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "subcategory",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "caller_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "resolved_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "resolved_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "child_incidents",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "incident_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "notify",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "problem_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "caused_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reopen_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rfc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Incidents"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "first_discovered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "operational_status",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_discovered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_cc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "checked_in",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "attributes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "serial_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vendor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ip_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "support_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "asset",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "supported_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "invoice_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "managed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fault_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_in",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "justification",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "lease_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "monitor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_center",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "subcategory",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "can_print",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "start_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "discovery_source",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "schedule",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fqdn",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warranty_expiration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "owned_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "asset_tag",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "manufacturer",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "purchase_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "department",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "checked_out",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "unverified",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "skip_sync",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "po_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "gl_account",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "maintenance_schedule",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "install_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "dns_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mac_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "change_control",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "install_status",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Incidents"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Customer Payment Query",
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Customer Payment Query"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Customer Payment Query",
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Customer Payment Query"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "test publish datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:ATTRIBUTE",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "ATTRIBUTE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:BINFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "BINFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:CALCULATEDFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "CALCULATEDFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:COLUMNFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "COLUMNFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:COUNT",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "COUNT"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:DATASOURCEFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "DATASOURCEFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:DIMENSION",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "DIMENSION"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:GROUPFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "GROUPFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:HIERARCHYFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "HIERARCHYFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:MEASURE",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "MEASURE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:SETFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "SETFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:SUM",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "SUM"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:TagSheet3",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "TagSheet3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:YEAR",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "YEAR"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:tag on published datasource",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "tag on published datasource"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_no_hidden_assets_ingest"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json
new file mode 100644
index 0000000000000..044bc7b6d1e0b
--- /dev/null
+++ b/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json
@@ -0,0 +1,44426 @@
+[
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01"
+            },
+            "name": "default"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c"
+            },
+            "name": "Project 2"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d"
+            },
+            "name": "Samples"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Project"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9",
+                "permissions": "[{\"group\": \"AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer\", \"capabilities\": [\"Read\", \"ViewComments\"]}]"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995",
+            "name": "Email Performance by Campaign",
+            "description": "Description for Email Performance by Campaign"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea",
+                "permissions": "[{\"group\": \"AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer\", \"capabilities\": [\"Read\", \"ViewComments\"]}]"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619",
+            "name": "Dvdrental Workbook",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": [
+                {
+                    "tag": "urn:li:tag:TagSheet3"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf",
+                "permissions": "[{\"group\": \"AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer\", \"capabilities\": [\"Read\", \"ViewComments\"]}]"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605",
+            "name": "Executive Dashboard",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "tableau",
+                "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c",
+                "permissions": "[{\"group\": \"AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer\", \"capabilities\": [\"Read\", \"ViewComments\"]}]"
+            },
+            "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904",
+            "name": "Workbook published ds",
+            "description": ""
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:tableau"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workbook"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "ownerTypes": {},
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "chartUsageStatistics",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1638860400000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "viewsCount": 5
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {
+                            "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783"
+                        },
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent",
+                        "title": "Timeline - Sent",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List",
+                        "title": "Campaign List",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Rate",
+                        "nullable": false,
+                        "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary",
+                        "title": "Summary",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Rate",
+                        "nullable": false,
+                        "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)",
+                    "schemaField": {
+                        "fieldPath": "Measure Names",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)",
+                    "schemaField": {
+                        "fieldPath": "Measure Values",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign",
+                        "title": "Mobile - Sent by Campaign",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)",
+                    "schemaField": {
+                        "fieldPath": "Activity Date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)",
+                    "schemaField": {
+                        "fieldPath": "Click-to-Open",
+                        "nullable": false,
+                        "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                    "schemaField": {
+                        "fieldPath": "Clickthrough Emails",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                    "schemaField": {
+                        "fieldPath": "Delivered Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)",
+                    "schemaField": {
+                        "fieldPath": "Delivery Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)",
+                    "schemaField": {
+                        "fieldPath": "ID",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Click Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Email Delivered)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)",
+                    "schemaField": {
+                        "fieldPath": "Id (Activity - Open Email)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)",
+                    "schemaField": {
+                        "fieldPath": "Id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)",
+                    "schemaField": {
+                        "fieldPath": "Open Rate",
+                        "nullable": false,
+                        "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                    "schemaField": {
+                        "fieldPath": "Opened Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)",
+                    "schemaField": {
+                        "fieldPath": "Program Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                    "schemaField": {
+                        "fieldPath": "Sent Email",
+                        "nullable": false,
+                        "description": "formula: COUNTD([Id])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1",
+                        "title": "Sheet 1",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639772911000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)",
+                    "schemaField": {
+                        "fieldPath": "Custom SQL Query",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.ArrayType": {}
+                            }
+                        },
+                        "nativeDataType": "TABLE",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2",
+                        "title": "Sheet 2",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773415000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)",
+                    "schemaField": {
+                        "fieldPath": "Custom SQL Query",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.ArrayType": {}
+                            }
+                        },
+                        "nativeDataType": "TABLE",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)",
+                    "schemaField": {
+                        "fieldPath": "First Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)",
+                    "schemaField": {
+                        "fieldPath": "Last Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)",
+                    "schemaField": {
+                        "fieldPath": "amount",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)",
+                    "schemaField": {
+                        "fieldPath": "customer_id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)",
+                    "schemaField": {
+                        "fieldPath": "payment_date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:YEAR"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)",
+                    "schemaField": {
+                        "fieldPath": "rental_id",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3",
+                        "title": "Sheet 3",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1640375456000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:TagSheet3"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)",
+                    "schemaField": {
+                        "fieldPath": "Category",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)",
+                    "schemaField": {
+                        "fieldPath": "Customer Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)",
+                    "schemaField": {
+                        "fieldPath": "Segment",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests",
+                        "title": "Opened Requests",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)",
+                    "schemaField": {
+                        "fieldPath": "Total # Request",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Requests",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change",
+                        "title": "Top 10 Items by Requests and YoY Change",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)",
+                    "schemaField": {
+                        "fieldPath": "Name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems",
+                        "title": "Opened Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)",
+                    "schemaField": {
+                        "fieldPath": "Total # Problems",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Problems",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue",
+                        "title": "Overdue",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems",
+                        "title": "High and Critical Priority Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)",
+                    "schemaField": {
+                        "fieldPath": "% of critical and high priority",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change",
+                        "title": "Total Incidents by Category and YoY Change",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors",
+                        "title": "Known Errors",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)",
+                    "schemaField": {
+                        "fieldPath": "% of known error",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)",
+                    "schemaField": {
+                        "fieldPath": "Known error",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Problems with Related Incidents",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Related Incidents",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests",
+                        "title": "Overdue Requests",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident",
+                        "title": "AVG Time to Solve an Incident",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)",
+                    "schemaField": {
+                        "fieldPath": "Time to Close an Incident (seconds)",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)",
+                    "schemaField": {
+                        "fieldPath": "Time to Close an Incident",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F",
+                        "title": "Made SLA?",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)",
+                    "schemaField": {
+                        "fieldPath": "% made SLA",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)",
+                    "schemaField": {
+                        "fieldPath": "Made SLA",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority",
+                        "title": "Tooltip - Incident Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems",
+                        "title": "Overdue Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)",
+                    "schemaField": {
+                        "fieldPath": "% of Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "REAL",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority",
+                        "title": "Tooltip - Problem Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority",
+                        "title": "Tooltip - Request Breakdown by Priority",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems",
+                        "title": "Age of Active Problems",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)"
+                            },
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)",
+                    "schemaField": {
+                        "fieldPath": "Age of Problems",
+                        "nullable": false,
+                        "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)",
+                    "schemaField": {
+                        "fieldPath": "Time Span Breakdown",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents",
+                        "title": "Opened Incidents",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                    "schemaField": {
+                        "fieldPath": "Active",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)",
+                    "schemaField": {
+                        "fieldPath": "Category (Incident)",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                    "schemaField": {
+                        "fieldPath": "Closed",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)",
+                    "schemaField": {
+                        "fieldPath": "Current Year Total Cases",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)",
+                    "schemaField": {
+                        "fieldPath": "Due date",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                    "schemaField": {
+                        "fieldPath": "Max Year?",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.BooleanType": {}
+                            }
+                        },
+                        "nativeDataType": "BOOLEAN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                    "schemaField": {
+                        "fieldPath": "Number",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)",
+                    "schemaField": {
+                        "fieldPath": "Opened Month Tooltip",
+                        "nullable": false,
+                        "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                    "schemaField": {
+                        "fieldPath": "Opened",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.TimeType": {}
+                            }
+                        },
+                        "nativeDataType": "DATETIME",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                    "schemaField": {
+                        "fieldPath": "Overdue",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.StringType": {}
+                            }
+                        },
+                        "nativeDataType": "STRING",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)",
+                    "schemaField": {
+                        "fieldPath": "Priority",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DIMENSION"
+                                },
+                                {
+                                    "tag": "urn:li:tag:COLUMNFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:SUM"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)",
+                    "schemaField": {
+                        "fieldPath": "Total Active Incidents",
+                        "nullable": false,
+                        "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NumberType": {}
+                            }
+                        },
+                        "nativeDataType": "INTEGER",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:MEASURE"
+                                },
+                                {
+                                    "tag": "urn:li:tag:CALCULATEDFIELD"
+                                },
+                                {
+                                    "tag": "urn:li:tag:ATTRIBUTE"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
+            "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.chart.ChartInfo": {
+                        "customProperties": {},
+                        "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1",
+                        "title": "published sheet ds",
+                        "description": "",
+                        "lastModified": {
+                            "created": {
+                                "time": 1641951867000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642658093000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "inputs": [
+                            {
+                                "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "inputFields",
+    "aspect": {
+        "json": {
+            "fields": [
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)",
+                    "schemaField": {
+                        "fieldPath": "amount",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)",
+                    "schemaField": {
+                        "fieldPath": "customer_first_name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                },
+                {
+                    "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)",
+                    "schemaField": {
+                        "fieldPath": "staff_last_name",
+                        "nullable": false,
+                        "description": "",
+                        "type": {
+                            "type": {
+                                "com.linkedin.schema.NullType": {}
+                            }
+                        },
+                        "nativeDataType": "UNKNOWN",
+                        "recursive": false,
+                        "globalTags": {
+                            "tags": [
+                                {
+                                    "tag": "urn:li:tag:DATASOURCEFIELD"
+                                }
+                            ]
+                        },
+                        "isPartOfKey": false
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardUsageStatistics",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1638860400000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "viewsCount": 3
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {
+                            "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66"
+                        },
+                        "title": "Email Performance by Campaign",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+                            "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+                            "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+                            "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)"
+                        ],
+                        "datasets": [],
+                        "dashboards": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1640200234000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "dvd Rental Dashboard",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)"
+                        ],
+                        "datasets": [],
+                        "dashboards": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773866000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "Story 1",
+                        "description": "",
+                        "charts": [],
+                        "datasets": [],
+                        "dashboards": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639773866000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1642199995000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": {
+            "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
+                        "customProperties": {},
+                        "title": "Executive Dashboard",
+                        "description": "",
+                        "charts": [
+                            "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+                            "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+                            "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+                            "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+                            "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+                            "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+                            "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+                            "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+                            "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+                            "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+                            "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+                            "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+                            "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)"
+                        ],
+                        "datasets": [],
+                        "dashboards": [],
+                        "lastModified": {
+                            "created": {
+                                "time": 1639768450000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            },
+                            "lastModified": {
+                                "time": 1639768502000,
+                                "actor": "urn:li:corpuser:jawadqu@gmail.com"
+                            }
+                        },
+                        "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: COUNTD([Id])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-02-09T00:05:25Z",
+                            "extractLastUpdateTime": "2018-02-09T00:05:25Z"
+                        },
+                        "name": "Marketo",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Delivery Rate",
+                                "nullable": false,
+                                "description": "formula: ZN([Delivered Email]/[Sent Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Program ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated At",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workspace Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User Agent (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Predictive",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Clickthrough Rate",
+                                "nullable": false,
+                                "description": "formula: [Clickthrough Emails]/[Delivered Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Open Rate",
+                                "nullable": false,
+                                "description": "formula: ZN([Opened Email]/[Delivered Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sent Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivered Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Email Delivered)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Mobile Device",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created At",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Clickthrough Emails",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Click Email)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is Mobile Device (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing ID (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Click-to-Open",
+                                "nullable": false,
+                                "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Email",
+                                "nullable": false,
+                                "description": "formula: COUNTD([Id (Activity - Open Email)])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign Run ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Non Clicked Emails",
+                                "nullable": false,
+                                "description": "formula: [Opened Email]-[Clickthrough Emails]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Non Clickthrough Email",
+                                "nullable": false,
+                                "description": "formula: [Delivered Email]-[Clickthrough Emails]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead ID (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Program Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign ID (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity Date (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test Variant (Activity - Click Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Non Opened Email",
+                                "nullable": false,
+                                "description": "formula: [Delivered Email]-[Opened Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has Predictive (Activity - Email Delivered)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User Agent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Bounceback",
+                                "nullable": false,
+                                "description": "formula: [Sent Email] - [Delivered Email]",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice Number (Activity - Open Email)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                    "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Customer Payment Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Custom SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rental_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "actor+ (dvdrental)",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Address2",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Update",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "District",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "country, city",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Phone",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Update (Address)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Address Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Actor Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "actor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Last Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City Id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Superstore Datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Region (People)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "People",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID (Returns)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Orders",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit Ratio",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returns",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Regional Manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit (bin)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Top Customers by Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T19:35:39Z",
+                            "extractLastUpdateTime": "2018-01-18T19:35:39Z"
+                        },
+                        "name": "Requests",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Closed by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mobile Picture",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% made SLA",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN  [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Meta",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total # Request",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requested for date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ordered item link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible elsewhere",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price Frequency (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Update name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Special instructions",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Execution Plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible on Bundles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No search",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "List Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer update",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Guide",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Package",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Estimated Delivery",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Billable",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price Frequency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Recurring Price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fulfillment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Backordered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ignore price",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN  \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Replace on upgrade",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Context",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requested for",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Billable (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mobile Picture Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Protection policy",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Omit price in cart",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Catalogs",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Entitlement script",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Published version",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Preview link",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Price (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Stage (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Visible on Guides",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Requested Item) 1",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery time",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Stage",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Request",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolve Time",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Icon",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Display name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No proceed checkout",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Use cart layout",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "No order now",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Application",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Template",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Catalog",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Image",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workflow",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Class",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created from item design",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Roles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Hide price (mobile listings)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Availability",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Vendor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Picture",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan script",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Requested Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Requests",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Start closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Request state",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Request)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Catalog Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T20:21:33Z",
+                            "extractLastUpdateTime": "2018-01-18T20:21:33Z"
+                        },
+                        "name": "Problems",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "SLA due (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Default assignee",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Workaround",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time Span Breakdown",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF  [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Change request",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost center",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Roles",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total # Problems",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Exclude manager",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lucha",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problems with Related Incidents",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Related Incidents",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Source",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of known error",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problem state",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Known error",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group email",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Age of Problems",
+                                "nullable": false,
+                                "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of critical and high priority",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR  [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Include members",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Problem)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Problems",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Group)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-01-18T20:13:08Z",
+                            "extractLastUpdateTime": "2018-01-18T20:13:08Z"
+                        },
+                        "name": "Incidents",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Assignment group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Attributes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User input (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close code (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Migrated Data",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Status",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Notify (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Requires verification",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Maintenance schedule",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Escalation",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Warranty expiration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "GL account",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional assignee list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "First discovered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Asset",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "% of Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Skip sync",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "DNS Domain",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Caller (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Department",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolved by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Current Year Total Cases",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Close notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Managed by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Names",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "PO number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Short description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business resolve time (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Child Incidents (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "IP Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Asset tag",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due in",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Checked out",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fully qualified domain name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Installed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Purchased",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lease contract",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Vendor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Overdue",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN  \"Overdue\"\r\nEND\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval history",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent Incident (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost currency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "SLA due (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Impact (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Subcategory (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Monitor",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Serial number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Model ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Parent (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time to Close an Incident (seconds)",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Owned by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity due",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Invoice number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval set (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Duration",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Start date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ordered",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business duration (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order received",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discovery source",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Total Active Incidents",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:ATTRIBUTE"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Class",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Operational status",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Expected start (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work notes list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolve time (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reopen count (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created by (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned to (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Most recent discovery",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon reject (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Knowledge (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Max Year?",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Watch list (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery task",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Fault count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Caused by Change (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updated (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "MAC Address",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Approval",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Priority (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Urgency",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Company (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Additional comments (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Business service (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Schedule",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Supported by",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Configuration item (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Support group",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation display (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Justification",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Change Request (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Updates",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Incident state (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Made SLA (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Opened Month Tooltip",
+                                "nullable": false,
+                                "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Problem (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Measure Values",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Group list",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Checked in",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Severity (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number of Records",
+                                "nullable": false,
+                                "description": "formula: 1",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time worked (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Cost center",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work end (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Due date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Created (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Delivery plan",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Subcategory (Configuration Item)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sys ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments and Work notes (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Can Print",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Active",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "BOOLEAN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Number (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Follow up (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Task type (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Domain Path (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Closed",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Description (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Reassignment count",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Contact type (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assignment group (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Comments",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Work start (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Correlation ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Resolved (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Assigned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Time to Close an Incident",
+                                "nullable": false,
+                                "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Upon approval (Incident)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d",
+                    "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)"
+                    ],
+                    "transformOperation": "IDENTITY",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "test publish datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Custom SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DATASOURCEFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Workbook published ds"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "hasExtracts": "True",
+                            "extractLastRefreshTime": "2018-02-09T00:05:25Z",
+                            "extractLastUpdateTime": "2018-02-09T00:05:25Z"
+                        },
+                        "name": "New DataSource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Program ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Embedded Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9",
+                    "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": [
+                            {
+                                "tag": "urn:li:tag:tag on published datasource"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "test publish datasource",
+                        "description": "description for test publish datasource",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATETIME",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Published SQL Query",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Published Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)"
+                    ],
+                    "transformOperation": "HierarchyField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)"
+                    ],
+                    "transformOperation": "GroupField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)"
+                    ],
+                    "transformOperation": "HierarchyField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)"
+                    ],
+                    "transformOperation": "BinField",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)"
+                    ],
+                    "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])",
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)"
+                    ],
+                    "transformOperation": "SetField",
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Ownership": {
+                        "owners": [
+                            {
+                                "owner": "urn:li:corpuser:jawadqu@gmail.com",
+                                "type": "DATAOWNER"
+                            }
+                        ],
+                        "ownerTypes": {},
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        }
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Superstore Datasource",
+                        "description": "Description for Superstore dataset",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Top Customers by Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:SETFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returns",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit Ratio",
+                                "nullable": false,
+                                "description": "formula: SUM([Profit])/SUM([Sales])",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:CALCULATEDFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Orders",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit (bin)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:BINFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID (Returns)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Person",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Location",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNKNOWN",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:HIERARCHYFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "People",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                                    }
+                                },
+                                "nativeDataType": "TABLE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COUNT"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Manufacturer",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:GROUPFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region (People)",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "REAL",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:MEASURE"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:SUM"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:YEAR"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STRING",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:DIMENSION"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:COLUMNFIELD"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Published Data Source"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873",
+                    "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "NUMERIC",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rental_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n    ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                },
+                {
+                    "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1",
+                    "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "amount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "NUMERIC",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "payment_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "staff_last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n    ON p.customer_id = c.customer_id\nINNER JOIN staff s \n    ON p.staff_id = s.staff_id\nORDER BY payment_date",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b",
+                    "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.DataPlatformInstance": {
+                        "platform": "urn:li:dataPlatform:tableau"
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "seller_city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/SubProject1/AbcJoinWorkbook"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {},
+                        "name": "Custom SQL Query",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.ViewProperties": {
+                        "materialized": false,
+                        "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id",
+                        "viewLanguage": "SQL"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View",
+                "Custom SQL"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "SubProject1"
+                },
+                {
+                    "id": "AbcJoinWorkbook"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Link_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Mobile_Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User_Agent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Platform",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Is_Mobile_Device",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "User_Agent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Test_Variant",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_Run_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Activity_Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Mailing_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Has_Predictive",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Campaign_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Step_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Lead_ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Choice_Number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Email Performance by Campaign/Marketo"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "programName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "programId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "createdAt",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "workspaceName",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "updatedAt",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Email Performance by Campaign"
+                },
+                {
+                    "id": "Marketo"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address2",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I2",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "district",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Dvdrental Workbook"
+                },
+                {
+                    "id": "actor+ (dvdrental)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "last_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "DBTIMESTAMP",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "first_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "STR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "actor_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I4",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Dvdrental Workbook"
+                },
+                {
+                    "id": "actor+ (dvdrental)"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Person",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Returned",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/Samples/Superstore Datasource"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "Product ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Postal Code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "City",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "State",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer Name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Country/Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sales",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Segment",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Sub-Category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Profit",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Product Name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Customer ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Order ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Row ID",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "I8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Discount",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "R8",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Ship Mode",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "Region",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WSTR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Samples"
+                },
+                {
+                    "id": "Superstore Datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems",
+                            "/prod/tableau/default/Executive Dashboard/Requests",
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "requested_for",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "requested_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "request_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "stage",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "special_instructions",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "configuration_item",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_catalog",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "stage",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "estimated_delivery",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "context",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "billable",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_frequency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cat_item",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order_guide",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "backordered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "request",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Requests"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "sc_catalogs",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_picture_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "workflow",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_customer_update",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_standalone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_quantity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_scope",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "template",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_proceed_checkout",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "billable",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "meta",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ordered_item_link",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_ic_version",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "image",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_policy",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "roles",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "picture",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "list_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_order_now",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vendor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sc_ic_item_staging",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "entitlement_script",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "icon",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ignore_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "start_closed",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_package",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_replace_on_upgrade",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_FLOAT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "use_sc_layout",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "availability",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "custom_cart",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_picture",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_cart",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mobile_hide_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_time",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "no_search",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_frequency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "recurring_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan_script",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_bundle",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_update_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "visible_guide",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "preview",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "omit_price",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Requests"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "u_u",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "source",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "exclude_manager",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_center",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_lucha",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "u_lu2",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "roles",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "default_assignee",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "manager",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "include_members",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Problems"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "related_incidents",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rfc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "problem_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_around",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "known_error",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Problems"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "urgency",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "severity",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_service",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_set",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent_incident",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "subcategory",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "caller_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "resolved_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "group_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_display",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "resolved_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "parent",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "user_input",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "escalation",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval_history",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "impact",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "expected_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "additional_assignee_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "child_incidents",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cmdb_ci",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "incident_state",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "notify",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reassignment_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "contact_type",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "problem_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "opened_at",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_notes_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "priority",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "time_worked",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "caused_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_reject",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_task",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "knowledge",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "closed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_end",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reopen_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "work_start",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "made_sla",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "calendar_stc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "rfc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_plan",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "close_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "activity_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "active",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "business_duration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "follow_up",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments_and_work_notes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "upon_approval",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "watch_list",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sla_due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Incidents"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Executive Dashboard/Incidents"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "test",
+                        "platform": "urn:li:dataPlatform:tableau",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "first_discovered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "operational_status",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_discovered",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_cc",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "checked_in",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "attributes",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "serial_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vendor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ip_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "support_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "asset",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "supported_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "invoice_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "managed_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fault_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due_in",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "correlation_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "justification",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_class_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "comments",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "company",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "lease_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "monitor",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "cost_center",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "subcategory",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "delivery_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assignment_group",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "can_print",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "short_description",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "model_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "start_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "discovery_source",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_domain_path",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "assigned_to",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "schedule",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fqdn",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warranty_expiration",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "owned_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "asset_tag",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "manufacturer",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "purchase_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATE",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "location",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "department",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_updated_on",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "checked_out",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "unverified",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "skip_sync",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_BOOL",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "po_number",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "order_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "gl_account",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "maintenance_schedule",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "install_date",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "dns_domain",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_created_by",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "mac_address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "change_control",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_STRING",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "install_status",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "due",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_DATETIME",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sys_mod_count",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "WDC_INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Executive Dashboard"
+                },
+                {
+                    "id": "Incidents"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Customer Payment Query",
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Customer Payment Query"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/Customer Payment Query",
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "Customer Payment Query"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/tableau/default/test publish datasource"
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "default"
+                },
+                {
+                    "id": "test publish datasource"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:ATTRIBUTE",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "ATTRIBUTE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:BINFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "BINFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:CALCULATEDFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "CALCULATEDFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:COLUMNFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "COLUMNFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:COUNT",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "COUNT"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:DATASOURCEFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "DATASOURCEFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:DIMENSION",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "DIMENSION"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:GROUPFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "GROUPFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:HIERARCHYFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "HIERARCHYFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:MEASURE",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "MEASURE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:SETFIELD",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "SETFIELD"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:SUM",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "SUM"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:TagSheet3",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "TagSheet3"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:YEAR",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "YEAR"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:tag on published datasource",
+    "changeType": "UPSERT",
+    "aspectName": "tagKey",
+    "aspect": {
+        "json": {
+            "name": "tag on published datasource"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1638860400000,
+        "runId": "tableau-test",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_tableau_group_ingest"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json
index 53d4a802541c3..621dbc642f6b1 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json
@@ -870,8 +870,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -939,6 +939,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1221,6 +1226,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1789,6 +1799,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2409,6 +2424,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2925,6 +2945,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3077,6 +3102,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3604,6 +3634,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4112,6 +4147,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4588,6 +4628,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5093,6 +5138,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5485,6 +5535,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5851,6 +5906,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6246,6 +6306,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6693,6 +6758,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7111,6 +7181,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7451,6 +7526,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7817,6 +7897,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8079,6 +8164,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8526,6 +8616,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8869,6 +8964,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9238,6 +9338,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9630,6 +9735,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10077,6 +10187,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10227,8 +10342,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10277,6 +10392,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10384,6 +10504,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10489,6 +10614,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10608,6 +10738,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11038,6 +11173,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -12896,7 +13036,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
                     "type": "TRANSFORMED"
                 },
                 {
@@ -12904,7 +13044,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)",
                     "type": "TRANSFORMED"
                 }
             ],
@@ -12995,6 +13135,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13336,6 +13481,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14251,6 +14401,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15212,6 +15367,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22084,6 +22244,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26204,6 +26369,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31558,6 +31728,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31902,6 +32077,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32671,6 +32851,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json
index 4811e462acdb2..d5f3e86c86f16 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json
@@ -1043,8 +1043,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -1112,6 +1112,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1398,6 +1403,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1970,6 +1980,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2594,6 +2609,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3114,6 +3134,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3270,6 +3295,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3805,6 +3835,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4317,6 +4352,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4797,6 +4837,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5306,6 +5351,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5702,6 +5752,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6072,6 +6127,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6471,6 +6531,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6922,6 +6987,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7344,6 +7414,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7688,6 +7763,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8058,6 +8138,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8324,6 +8409,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8775,6 +8865,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9122,6 +9217,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9495,6 +9595,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9891,6 +9996,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10342,6 +10452,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10496,8 +10611,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10546,6 +10661,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/site2/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10657,6 +10777,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10766,6 +10891,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10889,6 +11019,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11323,6 +11458,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13284,6 +13424,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13629,6 +13774,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14548,6 +14698,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15513,6 +15668,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22389,6 +22549,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26513,6 +26678,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31871,6 +32041,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32219,6 +32394,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32996,6 +33176,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json
index 9c83fdbc1d879..b4dfc7b4885c7 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json
@@ -1043,8 +1043,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -1112,6 +1112,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1398,6 +1403,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1970,6 +1980,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2594,6 +2609,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3114,6 +3134,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3270,6 +3295,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3805,6 +3835,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4317,6 +4352,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4797,6 +4837,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5306,6 +5351,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5702,6 +5752,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6072,6 +6127,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6471,6 +6531,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6922,6 +6987,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7344,6 +7414,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7688,6 +7763,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8058,6 +8138,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8324,6 +8409,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8775,6 +8865,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9122,6 +9217,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9495,6 +9595,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9891,6 +9996,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10342,6 +10452,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10496,8 +10611,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10546,6 +10661,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10657,6 +10777,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10766,6 +10891,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10889,6 +11019,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11323,6 +11458,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13284,6 +13424,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13629,6 +13774,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14548,6 +14698,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15513,6 +15668,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22389,6 +22549,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26513,6 +26678,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31871,6 +32041,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32219,6 +32394,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32996,6 +33176,11 @@
                         "platform": "urn:li:dataPlatform:tableau"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json
index fdecf887b5305..19ce71da97572 100644
--- a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json
+++ b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json
@@ -817,8 +817,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 5
         }
@@ -887,6 +887,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1174,6 +1179,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -1747,6 +1757,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2372,6 +2387,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -2893,6 +2913,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3050,6 +3075,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -3587,6 +3617,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4100,6 +4135,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -4581,6 +4621,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5091,6 +5136,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5488,6 +5538,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -5859,6 +5914,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6259,6 +6319,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -6711,6 +6776,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7134,6 +7204,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7479,6 +7554,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -7850,6 +7930,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8117,6 +8202,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8569,6 +8659,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -8917,6 +9012,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9291,6 +9391,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -9688,6 +9793,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10140,6 +10250,11 @@
                             "actor": "urn:li:corpuser:unknown"
                         }
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
                 }
             ]
         }
@@ -10294,8 +10409,8 @@
         "json": {
             "timestampMillis": 1638860400000,
             "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
             },
             "viewsCount": 3
         }
@@ -10345,6 +10460,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10457,6 +10577,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10567,6 +10692,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -10691,6 +10821,11 @@
                         "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -11126,6 +11261,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13088,6 +13228,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -13434,6 +13579,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -14354,6 +14504,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -15320,6 +15475,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -22197,6 +22357,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -26322,6 +26487,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -31681,6 +31851,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32030,6 +32205,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
@@ -32809,6 +32989,11 @@
                         "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)"
                     }
                 },
+                {
+                    "com.linkedin.pegasus2avro.common.GlobalTags": {
+                        "tags": []
+                    }
+                },
                 {
                     "com.linkedin.pegasus2avro.common.BrowsePaths": {
                         "paths": [
diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
index 5a5552a78c56f..c3a8880bf20a0 100644
--- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
+++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
@@ -7,21 +7,27 @@
 
 import pytest
 from freezegun import freeze_time
+from pydantic import ValidationError
 from requests.adapters import ConnectionError
-from tableauserverclient import Server
+from tableauserverclient import PermissionsRule, Server
 from tableauserverclient.models import (
     DatasourceItem,
+    GroupItem,
     ProjectItem,
     SiteItem,
     ViewItem,
     WorkbookItem,
 )
+from tableauserverclient.models.reference_item import ResourceReference
 
-from datahub.configuration.source_common import DEFAULT_ENV
-from datahub.emitter.mce_builder import make_schema_field_urn
+from datahub.emitter.mce_builder import DEFAULT_ENV, make_schema_field_urn
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.source import TestConnectionReport
 from datahub.ingestion.run.pipeline import Pipeline, PipelineContext
+from datahub.ingestion.source.tableau import tableau_constant as c
 from datahub.ingestion.source.tableau.tableau import (
     TableauConfig,
+    TableauProject,
     TableauSiteSource,
     TableauSource,
     TableauSourceReport,
@@ -37,7 +43,7 @@
     FineGrainedLineageUpstreamType,
     UpstreamLineage,
 )
-from datahub.metadata.schema_classes import MetadataChangeProposalClass, UpstreamClass
+from datahub.metadata.schema_classes import UpstreamClass
 from tests.test_helpers import mce_helpers, test_connection_helpers
 from tests.test_helpers.state_helpers import (
     get_current_checkpoint_from_pipeline,
@@ -58,7 +64,8 @@
     "site": "acryl",
     "projects": ["default", "Project 2", "Samples"],
     "extract_project_hierarchy": False,
-    "page_size": 10,
+    "page_size": 1000,
+    "workbook_page_size": 1000,
     "ingest_tags": True,
     "ingest_owner": True,
     "ingest_tables_external": True,
@@ -131,6 +138,43 @@ def side_effect_project_data(*arg, **kwargs):
     return [project1, project2, project3, project4], mock_pagination
 
 
+def side_effect_group_data(*arg, **kwargs):
+    mock_pagination = mock.MagicMock()
+    mock_pagination.total_available = None
+
+    group1: GroupItem = GroupItem(
+        name="AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer"
+    )
+    group1._id = "79d02655-88e5-45a6-9f9b-eeaf5fe54903-group1"
+    group2: GroupItem = GroupItem(
+        name="AB_XY00-Tableau-Access_A_123_PROJECT_XY_Analyst"
+    )
+    group2._id = "79d02655-88e5-45a6-9f9b-eeaf5fe54903-group2"
+
+    return [group1, group2], mock_pagination
+
+
+def side_effect_workbook_permissions(*arg, **kwargs):
+    project_capabilities1 = {"Read": "Allow", "ViewComments": "Allow"}
+    reference: ResourceReference = ResourceReference(
+        id_="79d02655-88e5-45a6-9f9b-eeaf5fe54903-group1", tag_name="group"
+    )
+    rule1 = PermissionsRule(grantee=reference, capabilities=project_capabilities1)
+
+    project_capabilities2 = {
+        "Read": "Allow",
+        "ViewComments": "Allow",
+        "Delete": "Allow",
+        "Write": "Allow",
+    }
+    reference2: ResourceReference = ResourceReference(
+        id_="79d02655-88e5-45a6-9f9b-eeaf5fe54903-group2", tag_name="group"
+    )
+    rule2 = PermissionsRule(grantee=reference2, capabilities=project_capabilities2)
+
+    return [rule1, rule2]
+
+
 def side_effect_site_data(*arg, **kwargs):
     mock_pagination = mock.MagicMock()
     mock_pagination.total_available = None
@@ -237,7 +281,6 @@ def mock_sdk_client(
     datasources_side_effect: List[dict],
     sign_out_side_effect: List[dict],
 ) -> mock.MagicMock:
-
     mock_client = mock.Mock()
     mocked_metadata = mock.Mock()
     mocked_metadata.query.side_effect = side_effect_query_metadata_response
@@ -248,8 +291,10 @@ def mock_sdk_client(
     mock_client.views = mock.Mock()
     mock_client.projects = mock.Mock()
     mock_client.sites = mock.Mock()
+    mock_client.groups = mock.Mock()
 
     mock_client.projects.get.side_effect = side_effect_project_data
+    mock_client.groups.get.side_effect = side_effect_group_data
     mock_client.sites.get.side_effect = side_effect_site_data
     mock_client.sites.get_by_id.side_effect = side_effect_site_get_by_id
 
@@ -259,6 +304,11 @@ def mock_sdk_client(
 
     mock_client.workbooks = mock.Mock()
     mock_client.workbooks.get.side_effect = side_effect_workbook_data
+    workbook_mock = mock.create_autospec(WorkbookItem, instance=True)
+    type(workbook_mock).permissions = mock.PropertyMock(
+        return_value=side_effect_workbook_permissions()
+    )
+    mock_client.workbooks.get_by_id.return_value = workbook_mock
 
     mock_client.views.get.side_effect = side_effect_usage_stat
     mock_client.auth.sign_in.return_value = None
@@ -526,53 +576,28 @@ def test_extract_all_project(pytestconfig, tmp_path, mock_datahub_graph):
 def test_value_error_projects_and_project_pattern(
     pytestconfig, tmp_path, mock_datahub_graph
 ):
-    # Ingestion should raise ValueError
-    output_file_name: str = "tableau_project_pattern_precedence_mces.json"
-    golden_file_name: str = "tableau_project_pattern_precedence_mces_golden.json"
-
     new_config = config_source_default.copy()
     new_config["projects"] = ["default"]
     new_config["project_pattern"] = {"allow": ["^Samples$"]}
 
-    try:
-        tableau_ingest_common(
-            pytestconfig,
-            tmp_path,
-            mock_data(),
-            golden_file_name,
-            output_file_name,
-            mock_datahub_graph,
-            pipeline_config=new_config,
-        )
-    except Exception as e:
-        assert "projects is deprecated. Please use project_path_pattern only" in str(e)
+    with pytest.raises(
+        ValidationError,
+        match=r".*projects is deprecated. Please use project_path_pattern only.*",
+    ):
+        TableauConfig.parse_obj(new_config)
 
 
 def test_project_pattern_deprecation(pytestconfig, tmp_path, mock_datahub_graph):
-    # Ingestion should raise ValueError
-    output_file_name: str = "tableau_project_pattern_deprecation_mces.json"
-    golden_file_name: str = "tableau_project_pattern_deprecation_mces_golden.json"
-
     new_config = config_source_default.copy()
     del new_config["projects"]
     new_config["project_pattern"] = {"allow": ["^Samples$"]}
     new_config["project_path_pattern"] = {"allow": ["^Samples$"]}
 
-    try:
-        tableau_ingest_common(
-            pytestconfig,
-            tmp_path,
-            mock_data(),
-            golden_file_name,
-            output_file_name,
-            mock_datahub_graph,
-            pipeline_config=new_config,
-        )
-    except Exception as e:
-        assert (
-            "project_pattern is deprecated. Please use project_path_pattern only"
-            in str(e)
-        )
+    with pytest.raises(
+        ValidationError,
+        match=r".*project_pattern is deprecated. Please use project_path_pattern only*",
+    ):
+        TableauConfig.parse_obj(new_config)
 
 
 def test_project_path_pattern_allow(pytestconfig, tmp_path, mock_datahub_graph):
@@ -629,7 +654,8 @@ def test_tableau_ingest_with_platform_instance(
         "site": "acryl",
         "platform_instance": "acryl_site1",
         "projects": ["default", "Project 2"],
-        "page_size": 10,
+        "page_size": 1000,
+        "workbook_page_size": 1000,
         "ingest_tags": True,
         "ingest_owner": True,
         "ingest_tables_external": True,
@@ -939,11 +965,12 @@ def test_tableau_unsupported_csql():
         database_override_map={"production database": "prod"}
     )
 
-    def test_lineage_metadata(
+    def check_lineage_metadata(
         lineage, expected_entity_urn, expected_upstream_table, expected_cll
     ):
-        mcp = cast(MetadataChangeProposalClass, next(iter(lineage)).metadata)
-        assert mcp.aspect == UpstreamLineage(
+        mcp = cast(MetadataChangeProposalWrapper, list(lineage)[0].metadata)
+
+        expected = UpstreamLineage(
             upstreams=[
                 UpstreamClass(
                     dataset=expected_upstream_table,
@@ -966,6 +993,9 @@ def test_lineage_metadata(
         )
         assert mcp.entityUrn == expected_entity_urn
 
+        actual_aspect = mcp.aspect
+        assert actual_aspect == expected
+
     csql_urn = "urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)"
     expected_upstream_table = "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_bigquery_project.invent_dw.UserDetail,PROD)"
     expected_cll = {
@@ -979,6 +1009,7 @@ def test_lineage_metadata(
         ctx=context,
         platform="tableau",
         site=SiteItem(name="Site 1", content_url="site1"),
+        site_id="site1",
         report=TableauSourceReport(),
         server=Server("https://test-tableau-server.com"),
     )
@@ -996,7 +1027,7 @@ def test_lineage_metadata(
         },
         out_columns=[],
     )
-    test_lineage_metadata(
+    check_lineage_metadata(
         lineage=lineage,
         expected_entity_urn=csql_urn,
         expected_upstream_table=expected_upstream_table,
@@ -1014,7 +1045,7 @@ def test_lineage_metadata(
         },
         out_columns=[],
     )
-    test_lineage_metadata(
+    check_lineage_metadata(
         lineage=lineage,
         expected_entity_urn=csql_urn,
         expected_upstream_table=expected_upstream_table,
@@ -1151,8 +1182,117 @@ def test_site_name_pattern(pytestconfig, tmp_path, mock_datahub_graph):
 
 @freeze_time(FROZEN_TIME)
 @pytest.mark.integration
-def test_permission_mode_switched_error(pytestconfig, tmp_path, mock_datahub_graph):
+def test_permission_ingestion(pytestconfig, tmp_path, mock_datahub_graph):
+    enable_logging()
+    output_file_name: str = "tableau_permission_ingestion_mces.json"
+    golden_file_name: str = "tableau_permission_ingestion_mces_golden.json"
 
+    new_pipeline_config: Dict[Any, Any] = {
+        **config_source_default,
+        "permission_ingestion": {
+            "enable_workbooks": True,
+            "group_name_pattern": {"allow": ["^.*_Consumer$"]},
+        },
+    }
+    tableau_ingest_common(
+        pytestconfig,
+        tmp_path,
+        mock_data(),
+        golden_file_name,
+        output_file_name,
+        mock_datahub_graph,
+        pipeline_config=new_pipeline_config,
+        pipeline_name="test_tableau_group_ingest",
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_no_hidden_assets(pytestconfig, tmp_path, mock_datahub_graph):
+    enable_logging()
+    output_file_name: str = "tableau_no_hidden_assets_mces.json"
+    golden_file_name: str = "tableau_no_hidden_assets_mces_golden.json"
+
+    new_config = config_source_default.copy()
+    del new_config["projects"]
+    new_config["ingest_hidden_assets"] = False
+
+    tableau_ingest_common(
+        pytestconfig,
+        tmp_path,
+        mock_data(),
+        golden_file_name,
+        output_file_name,
+        mock_datahub_graph,
+        pipeline_config=new_config,
+        pipeline_name="test_tableau_no_hidden_assets_ingest",
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_ingest_tags_disabled(pytestconfig, tmp_path, mock_datahub_graph):
+    enable_logging()
+    output_file_name: str = "tableau_ingest_tags_disabled_mces.json"
+    golden_file_name: str = "tableau_ingest_tags_disabled_mces_golden.json"
+
+    new_config = config_source_default.copy()
+    new_config["ingest_tags"] = False
+
+    tableau_ingest_common(
+        pytestconfig,
+        tmp_path,
+        mock_data(),
+        golden_file_name,
+        output_file_name,
+        mock_datahub_graph,
+        pipeline_config=new_config,
+        pipeline_name="test_tableau_ingest_tags_disabled",
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_hidden_asset_tags(pytestconfig, tmp_path, mock_datahub_graph):
+    enable_logging()
+    output_file_name: str = "tableau_hidden_asset_tags_mces.json"
+    golden_file_name: str = "tableau_hidden_asset_tags_mces_golden.json"
+
+    new_config = config_source_default.copy()
+    del new_config["projects"]
+    new_config["tags_for_hidden_assets"] = ["hidden", "private"]
+
+    tableau_ingest_common(
+        pytestconfig,
+        tmp_path,
+        mock_data(),
+        golden_file_name,
+        output_file_name,
+        mock_datahub_graph,
+        pipeline_config=new_config,
+        pipeline_name="test_tableau_hidden_asset_tags_ingest",
+    )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_hidden_assets_without_ingest_tags(pytestconfig, tmp_path, mock_datahub_graph):
+    enable_logging()
+
+    new_config = config_source_default.copy()
+    new_config["tags_for_hidden_assets"] = ["hidden", "private"]
+    new_config["ingest_tags"] = False
+
+    with pytest.raises(
+        ValidationError,
+        match=r".*tags_for_hidden_assets is only allowed with ingest_tags enabled.*",
+    ):
+        TableauConfig.parse_obj(new_config)
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_permission_warning(pytestconfig, tmp_path, mock_datahub_graph):
     with mock.patch(
         "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph",
         mock_datahub_graph,
@@ -1174,6 +1314,7 @@ def test_permission_mode_switched_error(pytestconfig, tmp_path, mock_datahub_gra
                 config=mock.MagicMock(),
                 ctx=mock.MagicMock(),
                 site=mock.MagicMock(),
+                site_id=None,
                 server=mock_sdk.return_value,
                 report=reporter,
             )
@@ -1184,15 +1325,180 @@ def test_permission_mode_switched_error(pytestconfig, tmp_path, mock_datahub_gra
                 query_filter=mock.MagicMock(),
                 current_cursor=None,
                 retries_remaining=1,
+                fetch_size=10,
             )
 
             warnings = list(reporter.warnings)
 
-            assert len(warnings) == 1
+            assert len(warnings) == 2
 
-            assert warnings[0].title == "Derived Permission Error"
+            assert warnings[0].title == "Insufficient Permissions"
 
-            assert warnings[0].message == (
+            assert warnings[1].title == "Derived Permission Error"
+
+            assert warnings[1].message == (
                 "Turn on your derived permissions. See for details "
                 "https://community.tableau.com/s/question/0D54T00000QnjHbSAJ/how-to-fix-the-permissionsmodeswitched-error"
             )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.parametrize(
+    "extract_project_hierarchy, allowed_projects",
+    [
+        (True, ["project1", "project4", "project3"]),
+        (False, ["project1", "project4"]),
+    ],
+)
+def test_extract_project_hierarchy(extract_project_hierarchy, allowed_projects):
+    context = PipelineContext(run_id="0", pipeline_name="test_tableau")
+
+    config_dict = config_source_default.copy()
+
+    del config_dict["stateful_ingestion"]
+    del config_dict["projects"]
+
+    config_dict["project_pattern"] = {
+        "allow": ["project1", "project4"],
+        "deny": ["project2"],
+    }
+
+    config_dict["extract_project_hierarchy"] = extract_project_hierarchy
+
+    config = TableauConfig.parse_obj(config_dict)
+
+    site_source = TableauSiteSource(
+        config=config,
+        ctx=context,
+        platform="tableau",
+        site=SiteItem(name="Site 1", content_url="site1"),
+        site_id="site1",
+        report=TableauSourceReport(),
+        server=Server("https://test-tableau-server.com"),
+    )
+
+    all_project_map: Dict[str, TableauProject] = {
+        "p1": TableauProject(
+            id="1",
+            name="project1",
+            path=[],
+            parent_id=None,
+            parent_name=None,
+            description=None,
+        ),
+        "p2": TableauProject(
+            id="2",
+            name="project2",
+            path=[],
+            parent_id="1",
+            parent_name="project1",
+            description=None,
+        ),
+        "p3": TableauProject(
+            id="3",
+            name="project3",
+            path=[],
+            parent_id="1",
+            parent_name="project1",
+            description=None,
+        ),
+        "p4": TableauProject(
+            id="4",
+            name="project4",
+            path=[],
+            parent_id=None,
+            parent_name=None,
+            description=None,
+        ),
+    }
+
+    site_source._init_tableau_project_registry(all_project_map)
+
+    assert allowed_projects == [
+        project.name for project in site_source.tableau_project_registry.values()
+    ]
+
+
+@pytest.mark.integration
+def test_connection_report_test(requests_mock):
+    server_info_response = """
+        <tsResponse xmlns:t="http://tableau.com/api">
+            <t:serverInfo>
+                <t:productVersion build="build-number">foo</t:productVersion>
+                <t:restApiVersion>2.4</t:restApiVersion>
+            </t:serverInfo>
+        </tsResponse>
+
+    """
+
+    requests_mock.register_uri(
+        "GET",
+        "https://do-not-connect/api/2.4/serverInfo",
+        text=server_info_response,
+        status_code=200,
+        headers={"Content-Type": "application/xml"},
+    )
+
+    signin_response = """
+        <tsResponse xmlns:t="http://tableau.com/api">
+            <t:credentials token="fake_token">
+                <t:site id="fake_site_luid" contentUrl="fake_site_content_url"/>
+                <t:user id="fake_user_id"/>
+            </t:credentials>
+        </tsResponse>
+    """
+
+    requests_mock.register_uri(
+        "POST",
+        "https://do-not-connect/api/2.4/auth/signin",
+        text=signin_response,
+        status_code=200,
+        headers={"Content-Type": "application/xml"},
+    )
+
+    user_by_id_response = """
+        <tsResponse xmlns:t="http://tableau.com/api">
+          <t:user id="user-id" name="foo@abc.com" siteRole="SiteAdministratorExplorer" />
+        </tsResponse>
+    """
+
+    requests_mock.register_uri(
+        "GET",
+        "https://do-not-connect/api/2.4/sites/fake_site_luid/users/fake_user_id",
+        text=user_by_id_response,
+        status_code=200,
+        headers={"Content-Type": "application/xml"},
+    )
+
+    report: TestConnectionReport = TableauSource.test_connection(config_source_default)
+
+    assert report
+    assert report.capability_report
+    assert report.capability_report.get(c.SITE_PERMISSION)
+    assert report.capability_report[c.SITE_PERMISSION].capable
+
+    # Role other than SiteAdministratorExplorer
+    user_by_id_response = """
+        <tsResponse xmlns:t="http://tableau.com/api">
+          <t:user id="user-id" name="foo@abc.com" siteRole="Explorer" />
+        </tsResponse>
+    """
+
+    requests_mock.register_uri(
+        "GET",
+        "https://do-not-connect/api/2.4/sites/fake_site_luid/users/fake_user_id",
+        text=user_by_id_response,
+        status_code=200,
+        headers={"Content-Type": "application/xml"},
+    )
+
+    report = TableauSource.test_connection(config_source_default)
+
+    assert report
+    assert report.capability_report
+    assert report.capability_report.get(c.SITE_PERMISSION)
+    assert report.capability_report[c.SITE_PERMISSION].capable is False
+    assert (
+        report.capability_report[c.SITE_PERMISSION].failure_reason
+        == "The user does not have the `Site Administrator Explorer` role. Their current role is Explorer."
+    )
diff --git a/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py b/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py
index c078f1b77fd1b..b8b0563a1d24e 100644
--- a/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py
+++ b/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py
@@ -282,7 +282,6 @@ def register_mock_data(workspace_client):
 
 
 def mock_hive_sql(query):
-
     if query == "DESCRIBE EXTENDED `bronze_kambi`.`bet` betStatusId":
         return [
             ("col_name", "betStatusId"),
diff --git a/metadata-ingestion/tests/performance/snowflake/test_snowflake.py b/metadata-ingestion/tests/performance/snowflake/test_snowflake.py
index 73b7790b62e9e..5042c78c2e7b9 100644
--- a/metadata-ingestion/tests/performance/snowflake/test_snowflake.py
+++ b/metadata-ingestion/tests/performance/snowflake/test_snowflake.py
@@ -16,7 +16,6 @@
 
 
 def run_test():
-
     with mock.patch("snowflake.connector.connect") as mock_connect:
         sf_connection = mock.MagicMock()
         sf_cursor = mock.MagicMock()
diff --git a/metadata-ingestion/tests/test_helpers/docker_helpers.py b/metadata-ingestion/tests/test_helpers/docker_helpers.py
index 20aec975787e4..d0e943bbe63da 100644
--- a/metadata-ingestion/tests/test_helpers/docker_helpers.py
+++ b/metadata-ingestion/tests/test_helpers/docker_helpers.py
@@ -4,10 +4,10 @@
 
 import pytest
 
-from datahub.testing.docker_utils import (  # noqa: F401
-    docker_compose_runner,
-    is_responsive,
-    wait_for_port,
+from datahub.testing.docker_utils import (  # noqa: F401,I250
+    docker_compose_runner as docker_compose_runner,
+    is_responsive as is_responsive,
+    wait_for_port as wait_for_port,
 )
 
 logger = logging.getLogger(__name__)
diff --git a/metadata-ingestion/tests/test_helpers/mce_helpers.py b/metadata-ingestion/tests/test_helpers/mce_helpers.py
index 9ee4642bfe6eb..0105e6d596970 100644
--- a/metadata-ingestion/tests/test_helpers/mce_helpers.py
+++ b/metadata-ingestion/tests/test_helpers/mce_helpers.py
@@ -17,15 +17,16 @@
     Union,
 )
 
+import pytest
+
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.sink.file import write_metadata_file
 from datahub.metadata.schema_classes import MetadataChangeEventClass
+from datahub.metadata.urns import Urn
 from datahub.testing.compare_metadata_json import (
     assert_metadata_files_equal,
     load_json_file,
 )
-from datahub.utilities.urns.urn import Urn
-from tests.test_helpers.type_helpers import PytestConfig
 
 logger = logging.getLogger(__name__)
 
@@ -77,11 +78,12 @@ def clean_nones(value):
 
 
 def check_golden_file(
-    pytestconfig: PytestConfig,
+    pytestconfig: pytest.Config,
     output_path: Union[str, os.PathLike],
     golden_path: Union[str, os.PathLike],
     ignore_paths: Sequence[str] = (),
     ignore_paths_v2: Sequence[str] = (),
+    ignore_order: bool = True,
 ) -> None:
     update_golden = pytestconfig.getoption("--update-golden-files")
     copy_output = pytestconfig.getoption("--copy-output-files")
@@ -92,14 +94,16 @@ def check_golden_file(
         copy_output=copy_output,
         ignore_paths=ignore_paths,
         ignore_paths_v2=ignore_paths_v2,
+        ignore_order=ignore_order,
     )
 
 
 def check_goldens_stream(
-    pytestconfig: PytestConfig,
+    pytestconfig: pytest.Config,
     outputs: List,
     golden_path: Union[str, os.PathLike],
     ignore_paths: Sequence[str] = (),
+    ignore_order: bool = True,
 ) -> None:
     with tempfile.NamedTemporaryFile() as f:
         write_metadata_file(pathlib.Path(f.name), outputs)
@@ -109,6 +113,7 @@ def check_goldens_stream(
             output_path=f.name,
             golden_path=golden_path,
             ignore_paths=ignore_paths,
+            ignore_order=ignore_order,
         )
 
 
@@ -318,7 +323,7 @@ def assert_entity_mce_aspect(
 ) -> int:
     # TODO: Replace with read_metadata_file()
     test_output = load_json_file(file)
-    entity_type = Urn.create_from_string(entity_urn).get_type()
+    entity_type = Urn.from_string(entity_urn).get_type()
     assert isinstance(test_output, list)
     # mce urns
     mces: List[MetadataChangeEventClass] = [
@@ -341,7 +346,7 @@ def assert_entity_mcp_aspect(
 ) -> int:
     # TODO: Replace with read_metadata_file()
     test_output = load_json_file(file)
-    entity_type = Urn.create_from_string(entity_urn).get_type()
+    entity_type = Urn.from_string(entity_urn).get_type()
     assert isinstance(test_output, list)
     # mcps that match entity_urn
     mcps: List[MetadataChangeProposalWrapper] = [
diff --git a/metadata-ingestion/tests/test_helpers/state_helpers.py b/metadata-ingestion/tests/test_helpers/state_helpers.py
index 76f2ab283790f..f68aef742fc73 100644
--- a/metadata-ingestion/tests/test_helpers/state_helpers.py
+++ b/metadata-ingestion/tests/test_helpers/state_helpers.py
@@ -1,5 +1,5 @@
 import types
-from typing import Any, Dict, Optional, Type, cast
+from typing import Any, Callable, Dict, Optional, Type, cast
 from unittest.mock import MagicMock, create_autospec
 
 import pytest
@@ -10,6 +10,7 @@
     IngestionCheckpointingProviderBase,
 )
 from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.graph.config import DatahubClientConfig
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.ingestion.source.state.checkpoint import Checkpoint
 from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState
@@ -101,6 +102,13 @@ def monkey_patch_get_latest_timeseries_value(
     return mock_datahub_graph_ctx.mock_graph
 
 
+@pytest.fixture
+def mock_datahub_graph_instance(
+    mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph]
+) -> DataHubGraph:
+    return mock_datahub_graph(DatahubClientConfig(server="http://fake.domain.local"))
+
+
 def get_current_checkpoint_from_pipeline(
     pipeline: Pipeline,
 ) -> Optional[Checkpoint[GenericCheckpointState]]:
diff --git a/metadata-ingestion/tests/test_helpers/type_helpers.py b/metadata-ingestion/tests/test_helpers/type_helpers.py
index 154960bbf7fc4..3a2215ed81ca9 100644
--- a/metadata-ingestion/tests/test_helpers/type_helpers.py
+++ b/metadata-ingestion/tests/test_helpers/type_helpers.py
@@ -1,12 +1,5 @@
 from typing import Optional, TypeVar
 
-# The current PytestConfig solution is somewhat ugly and not ideal.
-# However, it is currently the best solution available, as the type itself is not
-# exported: https://docs.pytest.org/en/stable/reference.html#config.
-# As pytest's type support improves, this will likely change.
-# TODO: revisit pytestconfig as https://github.com/pytest-dev/pytest/issues/7469 progresses.
-from _pytest.config import Config as PytestConfig  # noqa: F401
-
 _T = TypeVar("_T")
 
 
diff --git a/metadata-ingestion/tests/unit/api/__init__.py b/metadata-ingestion/tests/unit/api/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/unit/api/entities/common/test_serialized_value.py b/metadata-ingestion/tests/unit/api/entities/common/test_serialized_value.py
index c9f16bbcef6fc..a72087376b78a 100644
--- a/metadata-ingestion/tests/unit/api/entities/common/test_serialized_value.py
+++ b/metadata-ingestion/tests/unit/api/entities/common/test_serialized_value.py
@@ -10,7 +10,6 @@ class MyTestModel(BaseModel):
 
 
 def test_base_model():
-
     test_base_model = MyTestModel(
         test_string_field="test_string_field",
         test_int_field=42,
@@ -31,7 +30,6 @@ def test_base_model():
 
 
 def test_dictwrapper():
-
     from datahub.metadata.schema_classes import DatasetPropertiesClass
 
     dataset_properties = DatasetPropertiesClass(
@@ -58,7 +56,6 @@ def test_dictwrapper():
 
 
 def test_raw_dictionary():
-
     test_object = {
         "test_string_field": "test_string_field",
         "test_int_field": 42,
diff --git a/metadata-ingestion/tests/unit/api/entities/platformresource/test_platform_resource.py b/metadata-ingestion/tests/unit/api/entities/platformresource/test_platform_resource.py
index e6c9a9466d62b..6a03f511fa51c 100644
--- a/metadata-ingestion/tests/unit/api/entities/platformresource/test_platform_resource.py
+++ b/metadata-ingestion/tests/unit/api/entities/platformresource/test_platform_resource.py
@@ -4,9 +4,12 @@
 
 import datahub.metadata.schema_classes as models
 from datahub.api.entities.platformresource.platform_resource import (
+    ElasticPlatformResourceQuery,
     PlatformResource,
     PlatformResourceKey,
+    PlatformResourceSearchFields,
 )
+from datahub.utilities.search_utils import LogicalOperator
 
 
 def test_platform_resource_dict():
@@ -179,3 +182,14 @@ class TestModel(BaseModel):
     ).encode("utf-8")
     assert platform_resource_info_mcp.aspect.value.schemaType == "JSON"
     assert platform_resource_info_mcp.aspect.value.schemaRef == TestModel.__name__
+
+
+def test_platform_resource_filters():
+    query = (
+        ElasticPlatformResourceQuery.create_from()
+        .group(LogicalOperator.AND)
+        .add_field_match(PlatformResourceSearchFields.PRIMARY_KEY, "test_1")
+        .add_field_match(PlatformResourceSearchFields.RESOURCE_TYPE, "server")
+        .end()
+    )
+    assert query.build() == '(primaryKey:"test_1" AND resourceType:"server")'
diff --git a/metadata-ingestion/tests/unit/api/entities/structuredproperties/__init__.py b/metadata-ingestion/tests/unit/api/entities/structuredproperties/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/unit/api/entities/structuredproperties/example_structured_properties_golden.json b/metadata-ingestion/tests/unit/api/entities/structuredproperties/example_structured_properties_golden.json
new file mode 100644
index 0000000000000..29386ece7b0ca
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/entities/structuredproperties/example_structured_properties_golden.json
@@ -0,0 +1,194 @@
+[
+{
+    "entityType": "structuredProperty",
+    "entityUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
+    "changeType": "UPSERT",
+    "aspectName": "propertyDefinition",
+    "aspect": {
+        "json": {
+            "qualifiedName": "io.acryl.privacy.retentionTime",
+            "displayName": "Retention Time",
+            "valueType": "urn:li:dataType:datahub.number",
+            "allowedValues": [
+                {
+                    "value": {
+                        "string": "30"
+                    },
+                    "description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
+                },
+                {
+                    "value": {
+                        "string": "90"
+                    },
+                    "description": "Use this for datasets that drive monthly reporting but contain pii"
+                },
+                {
+                    "value": {
+                        "string": "365"
+                    },
+                    "description": "Use this for non-sensitive data that can be retained for longer"
+                }
+            ],
+            "cardinality": "MULTIPLE",
+            "entityTypes": [
+                "urn:li:entityType:datahub.dataset",
+                "urn:li:entityType:datahub.dataFlow"
+            ],
+            "description": "Retention Time is used to figure out how long to retain records in a dataset",
+            "immutable": false
+        }
+    }
+},
+{
+    "entityType": "structuredProperty",
+    "entityUrn": "urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA",
+    "changeType": "UPSERT",
+    "aspectName": "propertyDefinition",
+    "aspect": {
+        "json": {
+            "qualifiedName": "io.acryl.dataManagement.replicationSLA",
+            "displayName": "Replication SLA",
+            "valueType": "urn:li:dataType:datahub.number",
+            "cardinality": "SINGLE",
+            "entityTypes": [
+                "urn:li:entityType:datahub.dataset"
+            ],
+            "description": "SLA for how long data can be delayed before replicating to the destination cluster",
+            "immutable": false
+        }
+    }
+},
+{
+    "entityType": "structuredProperty",
+    "entityUrn": "urn:li:structuredProperty:io.acryl.dataManagement.deprecationDate",
+    "changeType": "UPSERT",
+    "aspectName": "propertyDefinition",
+    "aspect": {
+        "json": {
+            "qualifiedName": "io.acryl.dataManagement.deprecationDate",
+            "displayName": "Deprecation Date",
+            "valueType": "urn:li:dataType:datahub.date",
+            "cardinality": "SINGLE",
+            "entityTypes": [
+                "urn:li:entityType:datahub.dataset",
+                "urn:li:entityType:datahub.dataFlow",
+                "urn:li:entityType:datahub.dataJob"
+            ],
+            "immutable": false
+        }
+    }
+},
+{
+    "entityType": "structuredProperty",
+    "entityUrn": "urn:li:structuredProperty:io.acryl.dataManagement.steward",
+    "changeType": "UPSERT",
+    "aspectName": "propertyDefinition",
+    "aspect": {
+        "json": {
+            "qualifiedName": "io.acryl.dataManagement.steward",
+            "displayName": "Steward",
+            "valueType": "urn:li:dataType:datahub.urn",
+            "typeQualifier": {
+                "allowedTypes": [
+                    "urn:li:entityType:datahub.corpuser",
+                    "urn:li:entityType:datahub.corpGroup"
+                ]
+            },
+            "cardinality": "SINGLE",
+            "entityTypes": [
+                "urn:li:entityType:datahub.dataset",
+                "urn:li:entityType:datahub.dataFlow",
+                "urn:li:entityType:datahub.dataJob"
+            ],
+            "immutable": false
+        }
+    }
+},
+{
+    "entityType": "structuredProperty",
+    "entityUrn": "urn:li:structuredProperty:io.acryl.dataManagement.certifier",
+    "changeType": "UPSERT",
+    "aspectName": "propertyDefinition",
+    "aspect": {
+        "json": {
+            "qualifiedName": "io.acryl.dataManagement.certifier",
+            "displayName": "Person Certifying the asset",
+            "valueType": "urn:li:dataType:datahub.urn",
+            "cardinality": "SINGLE",
+            "entityTypes": [
+                "urn:li:entityType:datahub.dataset",
+                "urn:li:entityType:datahub.schemaField"
+            ],
+            "immutable": false
+        }
+    }
+},
+{
+    "entityType": "structuredProperty",
+    "entityUrn": "urn:li:structuredProperty:io.acryl.dataManagement.team",
+    "changeType": "UPSERT",
+    "aspectName": "propertyDefinition",
+    "aspect": {
+        "json": {
+            "qualifiedName": "io.acryl.dataManagement.team",
+            "displayName": "Management team",
+            "valueType": "urn:li:dataType:datahub.string",
+            "cardinality": "SINGLE",
+            "entityTypes": [
+                "urn:li:entityType:datahub.dataset"
+            ],
+            "immutable": false
+        }
+    }
+},
+{
+    "entityType": "structuredProperty",
+    "entityUrn": "urn:li:structuredProperty:projectNames",
+    "changeType": "UPSERT",
+    "aspectName": "propertyDefinition",
+    "aspect": {
+        "json": {
+            "qualifiedName": "projectNames",
+            "displayName": "Project names",
+            "valueType": "urn:li:dataType:datahub.string",
+            "allowedValues": [
+                {
+                    "value": {
+                        "string": "Tracking"
+                    },
+                    "description": "test value 1 for project"
+                },
+                {
+                    "value": {
+                        "string": "DataHub"
+                    },
+                    "description": "test value 2 for project"
+                }
+            ],
+            "cardinality": "MULTIPLE",
+            "entityTypes": [
+                "urn:li:entityType:datahub.dataset"
+            ],
+            "immutable": false
+        }
+    }
+},
+{
+    "entityType": "structuredProperty",
+    "entityUrn": "urn:li:structuredProperty:namespace",
+    "changeType": "UPSERT",
+    "aspectName": "propertyDefinition",
+    "aspect": {
+        "json": {
+            "qualifiedName": "namespace",
+            "displayName": "Namespace",
+            "valueType": "urn:li:dataType:datahub.string",
+            "cardinality": "SINGLE",
+            "entityTypes": [
+                "urn:li:entityType:datahub.dataset"
+            ],
+            "immutable": false
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/api/entities/structuredproperties/test_structuredproperties.py b/metadata-ingestion/tests/unit/api/entities/structuredproperties/test_structuredproperties.py
new file mode 100644
index 0000000000000..e96b7c1f98437
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/entities/structuredproperties/test_structuredproperties.py
@@ -0,0 +1,38 @@
+import pathlib
+
+import pydantic
+import pytest
+
+from datahub.api.entities.structuredproperties.structuredproperties import (
+    StructuredProperties,
+    TypeQualifierAllowedTypes,
+)
+from tests.test_helpers.mce_helpers import check_goldens_stream
+
+RESOURCE_DIR = pathlib.Path(__file__).parent
+
+
+def test_type_validation() -> None:
+    with pytest.raises(pydantic.ValidationError):
+        TypeQualifierAllowedTypes(allowed_types=["thisdoesnotexist"])
+
+    types = TypeQualifierAllowedTypes(allowed_types=["dataset"])
+    assert types.allowed_types == ["urn:li:entityType:datahub.dataset"]
+
+
+def test_structuredproperties_load(pytestconfig: pytest.Config) -> None:
+    example_properties_file = (
+        pytestconfig.rootpath
+        / "examples/structured_properties/structured_properties.yaml"
+    )
+
+    properties = StructuredProperties.from_yaml(str(example_properties_file))
+    mcps = []
+    for property in properties:
+        mcps.extend(property.generate_mcps())
+
+    check_goldens_stream(
+        pytestconfig,
+        mcps,
+        golden_path=RESOURCE_DIR / "example_structured_properties_golden.json",
+    )
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/__init__.py b/metadata-ingestion/tests/unit/api/source_helpers/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_auto_browse_path_v2.py b/metadata-ingestion/tests/unit/api/source_helpers/test_auto_browse_path_v2.py
new file mode 100644
index 0000000000000..0ad777c577d70
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_auto_browse_path_v2.py
@@ -0,0 +1,448 @@
+from itertools import zip_longest
+from typing import Any, Dict, Iterable, List
+from unittest.mock import patch
+
+import datahub.metadata.schema_classes as models
+from datahub.emitter.mce_builder import (
+    make_container_urn,
+    make_dataplatform_instance_urn,
+    make_dataset_urn,
+)
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.mcp_builder import DatabaseKey, SchemaKey
+from datahub.ingestion.api.source_helpers import (
+    _prepend_platform_instance,
+    auto_browse_path_v2,
+    auto_status_aspect,
+)
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.sql.sql_utils import gen_schema_container
+from datahub.metadata.schema_classes import BrowsePathEntryClass, BrowsePathsV2Class
+
+
+def test_auto_browse_path_v2_gen_containers_threaded():
+    database_key = DatabaseKey(platform="snowflake", database="db")
+    schema_keys = [
+        SchemaKey(platform="snowflake", database="db", schema=f"schema_{i}")
+        for i in range(10)
+    ]
+
+    wus_per_schema = [
+        gen_schema_container(
+            schema=key.db_schema,
+            database=key.database,
+            sub_types=[],
+            database_container_key=database_key,
+            schema_container_key=key,
+        )
+        for key in schema_keys
+    ]
+    for wu in auto_browse_path_v2(_iterate_wus_round_robin(wus_per_schema)):
+        aspect = wu.get_aspect_of_type(BrowsePathsV2Class)
+        if aspect:
+            assert aspect.path == [
+                BrowsePathEntryClass(
+                    id=database_key.as_urn(), urn=database_key.as_urn()
+                )
+            ]
+
+
+def _iterate_wus_round_robin(
+    mcps_per_schema: List[Iterable[MetadataWorkUnit]],
+) -> Iterable[MetadataWorkUnit]:
+    # Simulate a potential ordering of MCPs when using thread pool to generate MCPs
+    for wus in zip_longest(*mcps_per_schema, fillvalue=None):
+        for wu in wus:
+            if wu is not None:
+                yield wu
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_by_container_hierarchy(telemetry_ping_mock):
+    structure = {
+        "one": {
+            "a": {"i": ["1", "2", "3"], "ii": ["4"]},
+            "b": {"iii": ["5", "6"]},
+        },
+        "two": {
+            "c": {"iv": [], "v": ["7", "8"]},
+        },
+        "three": {"d": {}},
+        "four": {},
+    }
+
+    wus = list(auto_status_aspect(_create_container_aspects(structure)))
+    assert (  # Sanity check
+        sum(bool(wu.get_aspect_of_type(models.StatusClass)) for wu in wus) == 21
+    )
+
+    new_wus = list(auto_browse_path_v2(wus))
+    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
+    assert (
+        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
+        == 21
+    )
+
+    paths = _get_browse_paths_from_wu(new_wus)
+    assert paths["one"] == []
+    assert (
+        paths["7"]
+        == paths["8"]
+        == _make_container_browse_path_entries(["two", "c", "v"])
+    )
+    assert paths["d"] == _make_container_browse_path_entries(["three"])
+    assert paths["i"] == _make_container_browse_path_entries(["one", "a"])
+
+    # Check urns emitted on demand -- not all at end
+    for urn in {wu.get_urn() for wu in new_wus}:
+        try:
+            idx = next(
+                i
+                for i, wu in enumerate(new_wus)
+                if wu.get_aspect_of_type(models.ContainerClass) and wu.get_urn() == urn
+            )
+        except StopIteration:
+            idx = next(
+                i
+                for i, wu in enumerate(new_wus)
+                if wu.get_aspect_of_type(models.StatusClass) and wu.get_urn() == urn
+            )
+        assert new_wus[idx + 1].get_aspect_of_type(
+            models.BrowsePathsV2Class
+        ) or new_wus[idx + 2].get_aspect_of_type(models.BrowsePathsV2Class)
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_ignores_urns_already_with(telemetry_ping_mock):
+    structure = {"a": {"b": {"c": {"d": ["e"]}}}}
+
+    wus = [
+        *auto_status_aspect(
+            _create_container_aspects(
+                structure,
+                other_aspects={
+                    "f": [
+                        models.BrowsePathsClass(paths=["/one/two"]),
+                        models.BrowsePathsV2Class(
+                            path=_make_browse_path_entries(["my", "path"])
+                        ),
+                    ],
+                    "c": [
+                        models.BrowsePathsV2Class(
+                            path=_make_container_browse_path_entries(["custom", "path"])
+                        )
+                    ],
+                },
+            ),
+        )
+    ]
+    new_wus = list(auto_browse_path_v2(wus))
+    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
+    assert (
+        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
+        == 6
+    )
+
+    paths = _get_browse_paths_from_wu(new_wus)
+    assert paths["a"] == []
+    assert paths["c"] == _make_container_browse_path_entries(["custom", "path"])
+    assert paths["f"] == _make_browse_path_entries(["my", "path"])
+    assert paths["d"] == _make_container_browse_path_entries(["custom", "path", "c"])
+    assert paths["e"] == _make_container_browse_path_entries(
+        ["custom", "path", "c", "d"]
+    )
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_with_platform_instance_and_source_browse_path_v2(
+    telemetry_ping_mock,
+):
+    structure = {"a": {"b": {"c": {"d": ["e"]}}}}
+
+    platform = "platform"
+    instance = "instance"
+
+    wus = [
+        *auto_status_aspect(
+            _create_container_aspects(
+                structure,
+                other_aspects={
+                    "a": [
+                        models.BrowsePathsV2Class(
+                            path=_make_browse_path_entries(["my", "path"]),
+                        ),
+                    ],
+                },
+            ),
+        )
+    ]
+    new_wus = list(
+        auto_browse_path_v2(wus, platform=platform, platform_instance=instance)
+    )
+    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
+    assert (
+        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
+        == 5
+    )
+
+    paths = _get_browse_paths_from_wu(new_wus)
+    assert paths["a"] == _with_platform_instance(
+        _make_browse_path_entries(["my", "path"]),
+    )
+    assert paths["b"] == _with_platform_instance(
+        [
+            *_make_browse_path_entries(["my", "path"]),
+            *_make_container_browse_path_entries(["a"]),
+        ],
+    )
+    assert paths["c"] == _with_platform_instance(
+        [
+            *_make_browse_path_entries(["my", "path"]),
+            *_make_container_browse_path_entries(["a", "b"]),
+        ],
+    )
+    assert paths["d"] == _with_platform_instance(
+        [
+            *_make_browse_path_entries(["my", "path"]),
+            *_make_container_browse_path_entries(["a", "b", "c"]),
+        ],
+    )
+    assert paths["e"] == _with_platform_instance(
+        [
+            *_make_browse_path_entries(["my", "path"]),
+            *_make_container_browse_path_entries(["a", "b", "c", "d"]),
+        ],
+    )
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_legacy_browse_path(telemetry_ping_mock):
+    platform = "platform"
+    env = "PROD"
+    wus = [
+        MetadataChangeProposalWrapper(
+            entityUrn=make_dataset_urn(platform, "dataset-1", env),
+            aspect=models.BrowsePathsClass(["/one/two"]),
+        ).as_workunit(),
+        MetadataChangeProposalWrapper(
+            entityUrn=make_dataset_urn(platform, "dataset-2", env),
+            aspect=models.BrowsePathsClass([f"/{platform}/{env}/something"]),
+        ).as_workunit(),
+        MetadataChangeProposalWrapper(
+            entityUrn=make_dataset_urn(platform, "dataset-3", env),
+            aspect=models.BrowsePathsClass([f"/{platform}/one/two"]),
+        ).as_workunit(),
+    ]
+    new_wus = list(auto_browse_path_v2(wus, drop_dirs=["platform", "PROD", "unused"]))
+    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
+    assert len(new_wus) == 6
+    paths = _get_browse_paths_from_wu(new_wus)
+    assert (
+        paths["platform,dataset-1,PROD)"]
+        == paths["platform,dataset-3,PROD)"]
+        == _make_browse_path_entries(["one", "two"])
+    )
+    assert paths["platform,dataset-2,PROD)"] == _make_browse_path_entries(["something"])
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_container_over_legacy_browse_path(telemetry_ping_mock):
+    structure = {"a": {"b": ["c"]}}
+    wus = list(
+        auto_status_aspect(
+            _create_container_aspects(
+                structure,
+                other_aspects={"b": [models.BrowsePathsClass(paths=["/one/two"])]},
+            ),
+        )
+    )
+    new_wus = list(auto_browse_path_v2(wus))
+    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
+    assert (
+        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
+        == 3
+    )
+
+    paths = _get_browse_paths_from_wu(new_wus)
+    assert paths["a"] == []
+    assert paths["b"] == _make_container_browse_path_entries(["a"])
+    assert paths["c"] == _make_container_browse_path_entries(["a", "b"])
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_with_platform_instance(telemetry_ping_mock):
+    platform = "my_platform"
+    platform_instance = "my_instance"
+    platform_instance_urn = make_dataplatform_instance_urn(platform, platform_instance)
+    platform_instance_entry = models.BrowsePathEntryClass(
+        platform_instance_urn, platform_instance_urn
+    )
+
+    structure = {"a": {"b": ["c"]}}
+    wus = list(auto_status_aspect(_create_container_aspects(structure)))
+
+    new_wus = list(
+        auto_browse_path_v2(
+            wus,
+            platform=platform,
+            platform_instance=platform_instance,
+        )
+    )
+    assert telemetry_ping_mock.call_count == 0
+
+    assert (
+        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
+        == 3
+    )
+    paths = _get_browse_paths_from_wu(new_wus)
+    assert paths["a"] == [platform_instance_entry]
+    assert paths["b"] == [
+        platform_instance_entry,
+        *_make_container_browse_path_entries(["a"]),
+    ]
+    assert paths["c"] == [
+        platform_instance_entry,
+        *_make_container_browse_path_entries(["a", "b"]),
+    ]
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_invalid_batch_telemetry(telemetry_ping_mock):
+    structure = {"a": {"b": ["c"]}}
+    b_urn = make_container_urn("b")
+    wus = [
+        *_create_container_aspects(structure),
+        MetadataChangeProposalWrapper(  # Browse path for b separate from its Container aspect
+            entityUrn=b_urn,
+            aspect=models.BrowsePathsClass(paths=["/one/two"]),
+        ).as_workunit(),
+    ]
+    wus = list(auto_status_aspect(wus))
+
+    assert telemetry_ping_mock.call_count == 0
+    _ = list(auto_browse_path_v2(wus))
+    assert telemetry_ping_mock.call_count == 1
+    assert telemetry_ping_mock.call_args_list[0][0][0] == "incorrect_browse_path_v2"
+    assert telemetry_ping_mock.call_args_list[0][0][1]["num_out_of_order"] == 0
+    assert telemetry_ping_mock.call_args_list[0][0][1]["num_out_of_batch"] == 1
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_no_invalid_batch_telemetry_for_unrelated_aspects(
+    telemetry_ping_mock,
+):
+    structure = {"a": {"b": ["c"]}}
+    b_urn = make_container_urn("b")
+    wus = [
+        *_create_container_aspects(structure),
+        MetadataChangeProposalWrapper(  # Browse path for b separate from its Container aspect
+            entityUrn=b_urn,
+            aspect=models.ContainerPropertiesClass("container name"),
+        ).as_workunit(),
+    ]
+    wus = list(auto_status_aspect(wus))
+
+    assert telemetry_ping_mock.call_count == 0
+    _ = list(auto_browse_path_v2(wus))
+    assert telemetry_ping_mock.call_count == 0
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_invalid_order_telemetry(telemetry_ping_mock):
+    structure = {"a": {"b": ["c"]}}
+    wus = list(reversed(list(_create_container_aspects(structure))))
+    wus = list(auto_status_aspect(wus))
+
+    assert telemetry_ping_mock.call_count == 0
+    new_wus = list(auto_browse_path_v2(wus))
+    assert (
+        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
+        > 0
+    )
+    assert telemetry_ping_mock.call_count == 1
+    assert telemetry_ping_mock.call_args_list[0][0][0] == "incorrect_browse_path_v2"
+    assert telemetry_ping_mock.call_args_list[0][0][1]["num_out_of_order"] == 1
+    assert telemetry_ping_mock.call_args_list[0][0][1]["num_out_of_batch"] == 0
+
+
+@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
+def test_auto_browse_path_v2_dry_run(telemetry_ping_mock):
+    structure = {"a": {"b": ["c"]}}
+    wus = list(reversed(list(_create_container_aspects(structure))))
+    wus = list(auto_status_aspect(wus))
+
+    assert telemetry_ping_mock.call_count == 0
+    new_wus = list(auto_browse_path_v2(wus, dry_run=True))
+    assert wus == new_wus
+    assert (
+        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
+        == 0
+    )
+    assert telemetry_ping_mock.call_count == 1
+
+
+def _with_platform_instance(
+    path: List[models.BrowsePathEntryClass],
+) -> List[models.BrowsePathEntryClass]:
+    platform = "platform"
+    instance = "instance"
+    return _prepend_platform_instance(path, platform, instance)
+
+
+def _get_browse_paths_from_wu(
+    stream: Iterable[MetadataWorkUnit],
+) -> Dict[str, List[models.BrowsePathEntryClass]]:
+    paths = {}
+    for wu in stream:
+        browse_path_v2 = wu.get_aspect_of_type(models.BrowsePathsV2Class)
+        if browse_path_v2:
+            name = wu.get_urn().split(":")[-1]
+            paths[name] = browse_path_v2.path
+    return paths
+
+
+def _create_container_aspects(
+    d: Dict[str, Any],
+    other_aspects: Dict[str, List[models._Aspect]] = {},
+    root: bool = True,
+) -> Iterable[MetadataWorkUnit]:
+    for k, v in d.items():
+        urn = make_container_urn(k)
+        yield MetadataChangeProposalWrapper(
+            entityUrn=urn, aspect=models.StatusClass(removed=False)
+        ).as_workunit()
+
+        for aspect in other_aspects.pop(k, []):
+            yield MetadataChangeProposalWrapper(
+                entityUrn=urn, aspect=aspect
+            ).as_workunit()
+
+        for child in list(v):
+            yield MetadataChangeProposalWrapper(
+                entityUrn=make_container_urn(child),
+                aspect=models.ContainerClass(container=urn),
+            ).as_workunit()
+        if isinstance(v, dict):
+            yield from _create_container_aspects(
+                v, other_aspects=other_aspects, root=False
+            )
+
+    if root:
+        for k, v in other_aspects.items():
+            for aspect in v:
+                yield MetadataChangeProposalWrapper(
+                    entityUrn=make_container_urn(k), aspect=aspect
+                ).as_workunit()
+
+
+def _make_container_browse_path_entries(
+    path: List[str],
+) -> List[models.BrowsePathEntryClass]:
+    return [
+        models.BrowsePathEntryClass(id=make_container_urn(s), urn=make_container_urn(s))
+        for s in path
+    ]
+
+
+def _make_browse_path_entries(path: List[str]) -> List[models.BrowsePathEntryClass]:
+    return [models.BrowsePathEntryClass(id=s, urn=None) for s in path]
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_ensure_aspect_size.py b/metadata-ingestion/tests/unit/api/source_helpers/test_ensure_aspect_size.py
new file mode 100644
index 0000000000000..bdf1e0a2e0e86
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_ensure_aspect_size.py
@@ -0,0 +1,346 @@
+import json
+import time
+from unittest.mock import patch
+
+import pytest
+from freezegun.api import freeze_time
+
+from datahub.emitter.aspect import JSON_CONTENT_TYPE
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.rest_emitter import INGEST_MAX_PAYLOAD_BYTES
+from datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size import (
+    EnsureAspectSizeProcessor,
+)
+from datahub.ingestion.api.source import SourceReport
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
+from datahub.metadata.schema_classes import (
+    ChangeTypeClass,
+    DatasetFieldProfileClass,
+    DatasetProfileClass,
+    DatasetSnapshotClass,
+    GenericAspectClass,
+    MetadataChangeProposalClass,
+    NumberTypeClass,
+    OtherSchemaClass,
+    SchemaFieldClass,
+    SchemaFieldDataTypeClass,
+    SchemaMetadataClass,
+    StatusClass,
+    StringTypeClass,
+    SubTypesClass,
+)
+
+
+@pytest.fixture
+def processor():
+    return EnsureAspectSizeProcessor(SourceReport())
+
+
+def too_big_schema_metadata() -> SchemaMetadataClass:
+    fields = [
+        SchemaFieldClass(
+            "aaaa",
+            nativeDataType="int",
+            type=SchemaFieldDataTypeClass(type=NumberTypeClass()),
+        ),
+        SchemaFieldClass(
+            "bbbb",
+            nativeDataType="string",
+            type=SchemaFieldDataTypeClass(type=StringTypeClass()),
+        ),
+        SchemaFieldClass(
+            "cccc",
+            nativeDataType="int",
+            type=SchemaFieldDataTypeClass(type=NumberTypeClass()),
+        ),
+    ]
+    # simple int type field takes ~160 bytes in JSON representation, below is to assure we exceed the threshold
+    for f_no in range(1000):
+        fields.append(
+            SchemaFieldClass(
+                fieldPath=f"t{f_no}",
+                nativeDataType="int",
+                type=SchemaFieldDataTypeClass(type=NumberTypeClass()),
+                description=20000 * "a",
+            )
+        )
+
+    # adding small field to check whether it will still be present in the output
+    fields.append(
+        SchemaFieldClass(
+            "dddd",
+            nativeDataType="int",
+            type=SchemaFieldDataTypeClass(type=NumberTypeClass()),
+        )
+    )
+    return SchemaMetadataClass(
+        schemaName="abcdef",
+        version=1,
+        platform="s3",
+        hash="ABCDE1234567890",
+        platformSchema=OtherSchemaClass(rawSchema="aaa"),
+        fields=fields,
+    )
+
+
+def proper_schema_metadata() -> SchemaMetadataClass:
+    fields = [
+        SchemaFieldClass(
+            "aaaa",
+            nativeDataType="int",
+            type=SchemaFieldDataTypeClass(type=NumberTypeClass()),
+        ),
+        SchemaFieldClass(
+            "bbbb",
+            nativeDataType="string",
+            type=SchemaFieldDataTypeClass(type=StringTypeClass()),
+        ),
+        SchemaFieldClass(
+            "cccc",
+            nativeDataType="int",
+            type=SchemaFieldDataTypeClass(type=NumberTypeClass()),
+        ),
+    ]
+    return SchemaMetadataClass(
+        schemaName="abcdef",
+        version=1,
+        platform="s3",
+        hash="ABCDE1234567890",
+        platformSchema=OtherSchemaClass(rawSchema="aaa"),
+        fields=fields,
+    )
+
+
+def proper_dataset_profile() -> DatasetProfileClass:
+    sample_values = [
+        "23483295",
+        "234234",
+        "324234",
+        "12123",
+        "3150314",
+        "19231",
+        "211",
+        "93498",
+        "12837",
+        "73847",
+        "12434",
+        "33466",
+        "98785",
+        "4546",
+        "4547",
+        "342",
+        "11",
+        "34",
+        "444",
+        "38576",
+    ]
+    field_profiles = [
+        DatasetFieldProfileClass(fieldPath="a", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="b", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="c", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="d", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="e", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="f", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="g", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="h", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="i", sampleValues=sample_values),
+        DatasetFieldProfileClass(fieldPath="j", sampleValues=sample_values),
+    ]
+    return DatasetProfileClass(
+        timestampMillis=int(time.time()) * 1000, fieldProfiles=field_profiles
+    )
+
+
+@freeze_time("2023-01-02 00:00:00")
+def test_ensure_size_of_proper_dataset_profile(processor):
+    profile = proper_dataset_profile()
+    orig_repr = json.dumps(profile.to_obj())
+    processor.ensure_dataset_profile_size(
+        "urn:li:dataset:(s3, dummy_dataset, DEV)", profile
+    )
+    assert orig_repr == json.dumps(
+        profile.to_obj()
+    ), "Aspect was modified in case where workunit processor should have been no-op"
+
+
+@freeze_time("2023-01-02 00:00:00")
+def test_ensure_size_of_too_big_schema_metadata(processor):
+    schema = too_big_schema_metadata()
+    assert len(schema.fields) == 1004
+
+    processor.ensure_schema_metadata_size(
+        "urn:li:dataset:(s3, dummy_dataset, DEV)", schema
+    )
+    assert len(schema.fields) < 1004, "Schema has not been properly truncated"
+    assert schema.fields[-1].fieldPath == "dddd", "Small field was not added at the end"
+    # +100kb is completely arbitrary, but we are truncating the aspect based on schema fields size only, not total taken
+    # by other parameters of the aspect - it is reasonable approach though - schema fields is the only field in schema
+    # metadata which can be expected to grow out of control
+    assert (
+        len(json.dumps(schema.to_obj())) < INGEST_MAX_PAYLOAD_BYTES + 100000
+    ), "Aspect exceeded acceptable size"
+
+
+@freeze_time("2023-01-02 00:00:00")
+def test_ensure_size_of_proper_schema_metadata(processor):
+    schema = proper_schema_metadata()
+    orig_repr = json.dumps(schema.to_obj())
+    processor.ensure_schema_metadata_size(
+        "urn:li:dataset:(s3, dummy_dataset, DEV)", schema
+    )
+    assert orig_repr == json.dumps(
+        schema.to_obj()
+    ), "Aspect was modified in case where workunit processor should have been no-op"
+
+
+@freeze_time("2023-01-02 00:00:00")
+def test_ensure_size_of_too_big_dataset_profile(processor):
+    profile = proper_dataset_profile()
+    big_field = DatasetFieldProfileClass(
+        fieldPath="big",
+        sampleValues=20 * [(int(INGEST_MAX_PAYLOAD_BYTES / 20) - 10) * "a"],
+    )
+    assert profile.fieldProfiles
+    profile.fieldProfiles.insert(4, big_field)
+    processor.ensure_dataset_profile_size(
+        "urn:li:dataset:(s3, dummy_dataset, DEV)", profile
+    )
+
+    expected_profile = proper_dataset_profile()
+    reduced_field = DatasetFieldProfileClass(
+        fieldPath="big",
+        sampleValues=[],
+    )
+    assert expected_profile.fieldProfiles
+    expected_profile.fieldProfiles.insert(4, reduced_field)
+    assert json.dumps(profile.to_obj()) == json.dumps(
+        expected_profile.to_obj()
+    ), "Field 'big' was not properly removed from aspect due to its size"
+
+
+@freeze_time("2023-01-02 00:00:00")
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_schema_metadata_size"
+)
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_dataset_profile_size"
+)
+def test_wu_processor_triggered_by_data_profile_aspect(
+    ensure_dataset_profile_size_mock, ensure_schema_metadata_size_mock, processor
+):
+    ret = [  # noqa: F841
+        *processor.ensure_aspect_size(
+            [
+                MetadataChangeProposalWrapper(
+                    entityUrn="urn:li:dataset:(urn:li:dataPlatform:s3, dummy_name, DEV)",
+                    aspect=proper_dataset_profile(),
+                ).as_workunit()
+            ]
+        )
+    ]
+    ensure_dataset_profile_size_mock.assert_called_once()
+    ensure_schema_metadata_size_mock.assert_not_called()
+
+
+@freeze_time("2023-01-02 00:00:00")
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_schema_metadata_size"
+)
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_dataset_profile_size"
+)
+def test_wu_processor_triggered_by_data_profile_aspect_mcpc(
+    ensure_dataset_profile_size_mock, ensure_schema_metadata_size_mock, processor
+):
+    profile_aspect = proper_dataset_profile()
+    mcpc = MetadataWorkUnit(
+        id="test",
+        mcp_raw=MetadataChangeProposalClass(
+            entityType="dataset",
+            changeType=ChangeTypeClass.UPSERT,
+            entityUrn="urn:li:dataset:(urn:li:dataPlatform:s3, dummy_name, DEV)",
+            aspectName=DatasetProfileClass.ASPECT_NAME,
+            aspect=GenericAspectClass(
+                value=json.dumps(profile_aspect.to_obj()).encode(),
+                contentType=JSON_CONTENT_TYPE,
+            ),
+        ),
+    )
+    ret = [*processor.ensure_aspect_size([mcpc])]  # noqa: F841
+    ensure_dataset_profile_size_mock.assert_called_once()
+    ensure_schema_metadata_size_mock.assert_not_called()
+
+
+@freeze_time("2023-01-02 00:00:00")
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_schema_metadata_size"
+)
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_dataset_profile_size"
+)
+def test_wu_processor_triggered_by_data_profile_aspect_mce(
+    ensure_dataset_profile_size_mock, ensure_schema_metadata_size_mock, processor
+):
+    snapshot = DatasetSnapshotClass(
+        urn="urn:li:dataset:(urn:li:dataPlatform:s3, dummy_name, DEV)",
+        aspects=[proper_schema_metadata()],
+    )
+    mce = MetadataWorkUnit(
+        id="test", mce=MetadataChangeEvent(proposedSnapshot=snapshot)
+    )
+    ret = [*processor.ensure_aspect_size([mce])]  # noqa: F841
+    ensure_schema_metadata_size_mock.assert_called_once()
+    ensure_dataset_profile_size_mock.assert_not_called()
+
+
+@freeze_time("2023-01-02 00:00:00")
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_schema_metadata_size"
+)
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_dataset_profile_size"
+)
+def test_wu_processor_triggered_by_schema_metadata_aspect(
+    ensure_dataset_profile_size_mock, ensure_schema_metadata_size_mock, processor
+):
+    ret = [  # noqa: F841
+        *processor.ensure_aspect_size(
+            [
+                MetadataChangeProposalWrapper(
+                    entityUrn="urn:li:dataset:(urn:li:dataPlatform:s3, dummy_name, DEV)",
+                    aspect=proper_schema_metadata(),
+                ).as_workunit()
+            ]
+        )
+    ]
+    ensure_schema_metadata_size_mock.assert_called_once()
+    ensure_dataset_profile_size_mock.assert_not_called()
+
+
+@freeze_time("2023-01-02 00:00:00")
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_schema_metadata_size"
+)
+@patch(
+    "datahub.ingestion.api.auto_work_units.auto_ensure_aspect_size.EnsureAspectSizeProcessor.ensure_dataset_profile_size"
+)
+def test_wu_processor_not_triggered_by_unhandled_aspects(
+    ensure_dataset_profile_size_mock, ensure_schema_metadata_size_mock, processor
+):
+    ret = [  # noqa: F841
+        *processor.ensure_aspect_size(
+            [
+                MetadataChangeProposalWrapper(
+                    entityUrn="urn:li:dataset:(urn:li:dataPlatform:s3, dummy_name, DEV)",
+                    aspect=StatusClass(removed=False),
+                ).as_workunit(),
+                MetadataChangeProposalWrapper(
+                    entityUrn="urn:li:dataset:(urn:li:dataPlatform:s3, dummy_name, DEV)",
+                    aspect=SubTypesClass(typeNames=["table"]),
+                ).as_workunit(),
+            ]
+        )
+    ]
+    ensure_schema_metadata_size_mock.assert_not_called()
+    ensure_dataset_profile_size_mock.assert_not_called()
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
index cafca521ae014..c5c4a378894c3 100644
--- a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
@@ -104,7 +104,6 @@ def test_incremental_table_lineage(tmp_path, pytestconfig):
 
 
 def test_incremental_table_lineage_empty_upstreams(tmp_path, pytestconfig):
-
     urn = make_dataset_urn(platform, "dataset1")
     aspect = make_lineage_aspect(
         "dataset1",
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py b/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py
index 97f65f1bd6a5b..cdfd24554e5e5 100644
--- a/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py
@@ -1,25 +1,18 @@
 import logging
 from datetime import datetime
-from typing import Any, Dict, Iterable, List, Union
-from unittest.mock import patch
+from typing import List, Union
 
 import pytest
 from freezegun import freeze_time
 
 import datahub.metadata.schema_classes as models
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
-from datahub.emitter.mce_builder import (
-    make_container_urn,
-    make_dataplatform_instance_urn,
-    make_dataset_urn,
-)
+from datahub.emitter.mce_builder import make_dataset_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.auto_work_units.auto_dataset_properties_aspect import (
     auto_patch_last_modified,
 )
 from datahub.ingestion.api.source_helpers import (
-    _prepend_platform_instance,
-    auto_browse_path_v2,
     auto_empty_dataset_usage_statistics,
     auto_lowercase_urns,
     auto_status_aspect,
@@ -106,261 +99,6 @@ def test_auto_status_aspect():
     assert list(auto_status_aspect(initial_wu)) == expected
 
 
-def _create_container_aspects(
-    d: Dict[str, Any],
-    other_aspects: Dict[str, List[models._Aspect]] = {},
-    root: bool = True,
-) -> Iterable[MetadataWorkUnit]:
-    for k, v in d.items():
-        urn = make_container_urn(k)
-        yield MetadataChangeProposalWrapper(
-            entityUrn=urn, aspect=models.StatusClass(removed=False)
-        ).as_workunit()
-
-        for aspect in other_aspects.pop(k, []):
-            yield MetadataChangeProposalWrapper(
-                entityUrn=urn, aspect=aspect
-            ).as_workunit()
-
-        for child in list(v):
-            yield MetadataChangeProposalWrapper(
-                entityUrn=make_container_urn(child),
-                aspect=models.ContainerClass(container=urn),
-            ).as_workunit()
-        if isinstance(v, dict):
-            yield from _create_container_aspects(
-                v, other_aspects=other_aspects, root=False
-            )
-
-    if root:
-        for k, v in other_aspects.items():
-            for aspect in v:
-                yield MetadataChangeProposalWrapper(
-                    entityUrn=make_container_urn(k), aspect=aspect
-                ).as_workunit()
-
-
-def _make_container_browse_path_entries(
-    path: List[str],
-) -> List[models.BrowsePathEntryClass]:
-    return [
-        models.BrowsePathEntryClass(id=make_container_urn(s), urn=make_container_urn(s))
-        for s in path
-    ]
-
-
-def _make_browse_path_entries(path: List[str]) -> List[models.BrowsePathEntryClass]:
-    return [models.BrowsePathEntryClass(id=s, urn=None) for s in path]
-
-
-def prepend_platform_instance(
-    path: List[models.BrowsePathEntryClass],
-) -> List[models.BrowsePathEntryClass]:
-    platform = "platform"
-    instance = "instance"
-    return _prepend_platform_instance(path, platform, instance)
-
-
-def _get_browse_paths_from_wu(
-    stream: Iterable[MetadataWorkUnit],
-) -> Dict[str, List[models.BrowsePathEntryClass]]:
-    paths = {}
-    for wu in stream:
-        browse_path_v2 = wu.get_aspect_of_type(models.BrowsePathsV2Class)
-        if browse_path_v2:
-            name = wu.get_urn().split(":")[-1]
-            paths[name] = browse_path_v2.path
-    return paths
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_by_container_hierarchy(telemetry_ping_mock):
-    structure = {
-        "one": {
-            "a": {"i": ["1", "2", "3"], "ii": ["4"]},
-            "b": {"iii": ["5", "6"]},
-        },
-        "two": {
-            "c": {"iv": [], "v": ["7", "8"]},
-        },
-        "three": {"d": {}},
-        "four": {},
-    }
-
-    wus = list(auto_status_aspect(_create_container_aspects(structure)))
-    assert (  # Sanity check
-        sum(bool(wu.get_aspect_of_type(models.StatusClass)) for wu in wus) == 21
-    )
-
-    new_wus = list(auto_browse_path_v2(wus))
-    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
-    assert (
-        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
-        == 21
-    )
-
-    paths = _get_browse_paths_from_wu(new_wus)
-    assert paths["one"] == []
-    assert (
-        paths["7"]
-        == paths["8"]
-        == _make_container_browse_path_entries(["two", "c", "v"])
-    )
-    assert paths["d"] == _make_container_browse_path_entries(["three"])
-    assert paths["i"] == _make_container_browse_path_entries(["one", "a"])
-
-    # Check urns emitted on demand -- not all at end
-    for urn in {wu.get_urn() for wu in new_wus}:
-        try:
-            idx = next(
-                i
-                for i, wu in enumerate(new_wus)
-                if wu.get_aspect_of_type(models.ContainerClass) and wu.get_urn() == urn
-            )
-        except StopIteration:
-            idx = next(
-                i
-                for i, wu in enumerate(new_wus)
-                if wu.get_aspect_of_type(models.StatusClass) and wu.get_urn() == urn
-            )
-        assert new_wus[idx + 1].get_aspect_of_type(
-            models.BrowsePathsV2Class
-        ) or new_wus[idx + 2].get_aspect_of_type(models.BrowsePathsV2Class)
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_ignores_urns_already_with(telemetry_ping_mock):
-    structure = {"a": {"b": {"c": {"d": ["e"]}}}}
-
-    wus = [
-        *auto_status_aspect(
-            _create_container_aspects(
-                structure,
-                other_aspects={
-                    "f": [
-                        models.BrowsePathsClass(paths=["/one/two"]),
-                        models.BrowsePathsV2Class(
-                            path=_make_browse_path_entries(["my", "path"])
-                        ),
-                    ],
-                    "c": [
-                        models.BrowsePathsV2Class(
-                            path=_make_container_browse_path_entries(["custom", "path"])
-                        )
-                    ],
-                },
-            ),
-        )
-    ]
-    new_wus = list(auto_browse_path_v2(wus))
-    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
-    assert (
-        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
-        == 6
-    )
-
-    paths = _get_browse_paths_from_wu(new_wus)
-    assert paths["a"] == []
-    assert paths["c"] == _make_container_browse_path_entries(["custom", "path"])
-    assert paths["f"] == _make_browse_path_entries(["my", "path"])
-    assert paths["d"] == _make_container_browse_path_entries(["custom", "path", "c"])
-    assert paths["e"] == _make_container_browse_path_entries(
-        ["custom", "path", "c", "d"]
-    )
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_with_platform_instance_and_source_browse_path_v2(
-    telemetry_ping_mock,
-):
-    structure = {"a": {"b": {"c": {"d": ["e"]}}}}
-
-    platform = "platform"
-    instance = "instance"
-
-    wus = [
-        *auto_status_aspect(
-            _create_container_aspects(
-                structure,
-                other_aspects={
-                    "a": [
-                        models.BrowsePathsV2Class(
-                            path=_make_browse_path_entries(["my", "path"]),
-                        ),
-                    ],
-                },
-            ),
-        )
-    ]
-    new_wus = list(
-        auto_browse_path_v2(wus, platform=platform, platform_instance=instance)
-    )
-    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
-    assert (
-        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
-        == 5
-    )
-
-    paths = _get_browse_paths_from_wu(new_wus)
-    assert paths["a"] == prepend_platform_instance(
-        _make_browse_path_entries(["my", "path"]),
-    )
-    assert paths["b"] == prepend_platform_instance(
-        [
-            *_make_browse_path_entries(["my", "path"]),
-            *_make_container_browse_path_entries(["a"]),
-        ],
-    )
-    assert paths["c"] == prepend_platform_instance(
-        [
-            *_make_browse_path_entries(["my", "path"]),
-            *_make_container_browse_path_entries(["a", "b"]),
-        ],
-    )
-    assert paths["d"] == prepend_platform_instance(
-        [
-            *_make_browse_path_entries(["my", "path"]),
-            *_make_container_browse_path_entries(["a", "b", "c"]),
-        ],
-    )
-    assert paths["e"] == prepend_platform_instance(
-        [
-            *_make_browse_path_entries(["my", "path"]),
-            *_make_container_browse_path_entries(["a", "b", "c", "d"]),
-        ],
-    )
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_legacy_browse_path(telemetry_ping_mock):
-    platform = "platform"
-    env = "PROD"
-    wus = [
-        MetadataChangeProposalWrapper(
-            entityUrn=make_dataset_urn(platform, "dataset-1", env),
-            aspect=models.BrowsePathsClass(["/one/two"]),
-        ).as_workunit(),
-        MetadataChangeProposalWrapper(
-            entityUrn=make_dataset_urn(platform, "dataset-2", env),
-            aspect=models.BrowsePathsClass([f"/{platform}/{env}/something"]),
-        ).as_workunit(),
-        MetadataChangeProposalWrapper(
-            entityUrn=make_dataset_urn(platform, "dataset-3", env),
-            aspect=models.BrowsePathsClass([f"/{platform}/one/two"]),
-        ).as_workunit(),
-    ]
-    new_wus = list(auto_browse_path_v2(wus, drop_dirs=["platform", "PROD", "unused"]))
-    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
-    assert len(new_wus) == 6
-    paths = _get_browse_paths_from_wu(new_wus)
-    assert (
-        paths["platform,dataset-1,PROD)"]
-        == paths["platform,dataset-3,PROD)"]
-        == _make_browse_path_entries(["one", "two"])
-    )
-    assert paths["platform,dataset-2,PROD)"] == _make_browse_path_entries(["something"])
-
-
 def test_auto_lowercase_aspects():
     mcws = auto_workunit(
         [
@@ -430,142 +168,6 @@ def test_auto_lowercase_aspects():
     assert list(auto_lowercase_urns(mcws)) == expected
 
 
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_container_over_legacy_browse_path(telemetry_ping_mock):
-    structure = {"a": {"b": ["c"]}}
-    wus = list(
-        auto_status_aspect(
-            _create_container_aspects(
-                structure,
-                other_aspects={"b": [models.BrowsePathsClass(paths=["/one/two"])]},
-            ),
-        )
-    )
-    new_wus = list(auto_browse_path_v2(wus))
-    assert not telemetry_ping_mock.call_count, telemetry_ping_mock.call_args_list
-    assert (
-        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
-        == 3
-    )
-
-    paths = _get_browse_paths_from_wu(new_wus)
-    assert paths["a"] == []
-    assert paths["b"] == _make_container_browse_path_entries(["a"])
-    assert paths["c"] == _make_container_browse_path_entries(["a", "b"])
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_with_platform_instance(telemetry_ping_mock):
-    platform = "my_platform"
-    platform_instance = "my_instance"
-    platform_instance_urn = make_dataplatform_instance_urn(platform, platform_instance)
-    platform_instance_entry = models.BrowsePathEntryClass(
-        platform_instance_urn, platform_instance_urn
-    )
-
-    structure = {"a": {"b": ["c"]}}
-    wus = list(auto_status_aspect(_create_container_aspects(structure)))
-
-    new_wus = list(
-        auto_browse_path_v2(
-            wus,
-            platform=platform,
-            platform_instance=platform_instance,
-        )
-    )
-    assert telemetry_ping_mock.call_count == 0
-
-    assert (
-        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
-        == 3
-    )
-    paths = _get_browse_paths_from_wu(new_wus)
-    assert paths["a"] == [platform_instance_entry]
-    assert paths["b"] == [
-        platform_instance_entry,
-        *_make_container_browse_path_entries(["a"]),
-    ]
-    assert paths["c"] == [
-        platform_instance_entry,
-        *_make_container_browse_path_entries(["a", "b"]),
-    ]
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_invalid_batch_telemetry(telemetry_ping_mock):
-    structure = {"a": {"b": ["c"]}}
-    b_urn = make_container_urn("b")
-    wus = [
-        *_create_container_aspects(structure),
-        MetadataChangeProposalWrapper(  # Browse path for b separate from its Container aspect
-            entityUrn=b_urn,
-            aspect=models.BrowsePathsClass(paths=["/one/two"]),
-        ).as_workunit(),
-    ]
-    wus = list(auto_status_aspect(wus))
-
-    assert telemetry_ping_mock.call_count == 0
-    _ = list(auto_browse_path_v2(wus))
-    assert telemetry_ping_mock.call_count == 1
-    assert telemetry_ping_mock.call_args_list[0][0][0] == "incorrect_browse_path_v2"
-    assert telemetry_ping_mock.call_args_list[0][0][1]["num_out_of_order"] == 0
-    assert telemetry_ping_mock.call_args_list[0][0][1]["num_out_of_batch"] == 1
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_no_invalid_batch_telemetry_for_unrelated_aspects(
-    telemetry_ping_mock,
-):
-    structure = {"a": {"b": ["c"]}}
-    b_urn = make_container_urn("b")
-    wus = [
-        *_create_container_aspects(structure),
-        MetadataChangeProposalWrapper(  # Browse path for b separate from its Container aspect
-            entityUrn=b_urn,
-            aspect=models.ContainerPropertiesClass("container name"),
-        ).as_workunit(),
-    ]
-    wus = list(auto_status_aspect(wus))
-
-    assert telemetry_ping_mock.call_count == 0
-    _ = list(auto_browse_path_v2(wus))
-    assert telemetry_ping_mock.call_count == 0
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_invalid_order_telemetry(telemetry_ping_mock):
-    structure = {"a": {"b": ["c"]}}
-    wus = list(reversed(list(_create_container_aspects(structure))))
-    wus = list(auto_status_aspect(wus))
-
-    assert telemetry_ping_mock.call_count == 0
-    new_wus = list(auto_browse_path_v2(wus))
-    assert (
-        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
-        > 0
-    )
-    assert telemetry_ping_mock.call_count == 1
-    assert telemetry_ping_mock.call_args_list[0][0][0] == "incorrect_browse_path_v2"
-    assert telemetry_ping_mock.call_args_list[0][0][1]["num_out_of_order"] == 1
-    assert telemetry_ping_mock.call_args_list[0][0][1]["num_out_of_batch"] == 0
-
-
-@patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
-def test_auto_browse_path_v2_dry_run(telemetry_ping_mock):
-    structure = {"a": {"b": ["c"]}}
-    wus = list(reversed(list(_create_container_aspects(structure))))
-    wus = list(auto_status_aspect(wus))
-
-    assert telemetry_ping_mock.call_count == 0
-    new_wus = list(auto_browse_path_v2(wus, dry_run=True))
-    assert wus == new_wus
-    assert (
-        sum(bool(wu.get_aspect_of_type(models.BrowsePathsV2Class)) for wu in new_wus)
-        == 0
-    )
-    assert telemetry_ping_mock.call_count == 1
-
-
 @freeze_time("2023-01-02 00:00:00")
 def test_auto_empty_dataset_usage_statistics(caplog: pytest.LogCaptureFixture) -> None:
     has_urn = make_dataset_urn("my_platform", "has_aspect")
diff --git a/metadata-ingestion/tests/unit/test_apis.py b/metadata-ingestion/tests/unit/api/test_apis.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_apis.py
rename to metadata-ingestion/tests/unit/api/test_apis.py
diff --git a/metadata-ingestion/tests/unit/test_report.py b/metadata-ingestion/tests/unit/api/test_entity_filter_report.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_report.py
rename to metadata-ingestion/tests/unit/api/test_entity_filter_report.py
diff --git a/metadata-ingestion/tests/unit/test_pipeline.py b/metadata-ingestion/tests/unit/api/test_pipeline.py
similarity index 96%
rename from metadata-ingestion/tests/unit/test_pipeline.py
rename to metadata-ingestion/tests/unit/api/test_pipeline.py
index a462f28136797..fe3d3160b729a 100644
--- a/metadata-ingestion/tests/unit/test_pipeline.py
+++ b/metadata-ingestion/tests/unit/api/test_pipeline.py
@@ -33,7 +33,9 @@
 
 class TestPipeline:
     @patch("confluent_kafka.Consumer", autospec=True)
-    @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True)
+    @patch(
+        "datahub.ingestion.source.kafka.kafka.KafkaSource.get_workunits", autospec=True
+    )
     @patch("datahub.ingestion.sink.console.ConsoleSink.close", autospec=True)
     @freeze_time(FROZEN_TIME)
     def test_configure(self, mock_sink, mock_source, mock_consumer):
@@ -198,7 +200,9 @@ def test_configure_with_rest_sink_with_additional_props_initializes_graph(
         assert pipeline.ctx.graph.config.token == pipeline.config.sink.config["token"]
 
     @freeze_time(FROZEN_TIME)
-    @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True)
+    @patch(
+        "datahub.ingestion.source.kafka.kafka.KafkaSource.get_workunits", autospec=True
+    )
     def test_configure_with_file_sink_does_not_init_graph(self, mock_source, tmp_path):
         pipeline = Pipeline.create(
             {
@@ -224,9 +228,9 @@ def test_configure_with_file_sink_does_not_init_graph(self, mock_source, tmp_pat
     def test_run_including_fake_transformation(self):
         pipeline = Pipeline.create(
             {
-                "source": {"type": "tests.unit.test_pipeline.FakeSource"},
+                "source": {"type": "tests.unit.api.test_pipeline.FakeSource"},
                 "transformers": [
-                    {"type": "tests.unit.test_pipeline.AddStatusRemovedTransformer"}
+                    {"type": "tests.unit.api.test_pipeline.AddStatusRemovedTransformer"}
                 ],
                 "sink": {"type": "tests.test_helpers.sink_helpers.RecordingSink"},
                 "run_id": "pipeline_test",
@@ -253,7 +257,7 @@ def test_run_including_registered_transformation(self):
 
         pipeline = Pipeline.create(
             {
-                "source": {"type": "tests.unit.test_pipeline.FakeSource"},
+                "source": {"type": "tests.unit.api.test_pipeline.FakeSource"},
                 "transformers": [
                     {
                         "type": "simple_add_dataset_ownership",
@@ -297,7 +301,7 @@ def test_pipeline_return_code(self, tmp_path, source, strict_warnings, exit_code
 ---
 run_id: pipeline_test
 source:
-    type: tests.unit.test_pipeline.{source}
+    type: tests.unit.api.test_pipeline.{source}
     config: {{}}
 sink:
     type: console
@@ -379,7 +383,7 @@ def test_pipeline_return_code(self, tmp_path, source, strict_warnings, exit_code
     def test_pipeline_process_commits(self, commit_policy, source, should_commit):
         pipeline = Pipeline.create(
             {
-                "source": {"type": f"tests.unit.test_pipeline.{source}"},
+                "source": {"type": f"tests.unit.api.test_pipeline.{source}"},
                 "sink": {"type": "console"},
                 "run_id": "pipeline_test",
             }
diff --git a/metadata-ingestion/tests/unit/test_plugin_system.py b/metadata-ingestion/tests/unit/api/test_plugin_system.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_plugin_system.py
rename to metadata-ingestion/tests/unit/api/test_plugin_system.py
diff --git a/metadata-ingestion/tests/unit/api/test_report.py b/metadata-ingestion/tests/unit/api/test_source_report.py
similarity index 100%
rename from metadata-ingestion/tests/unit/api/test_report.py
rename to metadata-ingestion/tests/unit/api/test_source_report.py
diff --git a/metadata-ingestion/tests/unit/test_workunit.py b/metadata-ingestion/tests/unit/api/test_workunit.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_workunit.py
rename to metadata-ingestion/tests/unit/api/test_workunit.py
diff --git a/metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py
new file mode 100644
index 0000000000000..a1981ccf76791
--- /dev/null
+++ b/metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py
@@ -0,0 +1,358 @@
+import datetime
+from typing import Dict, List, Optional, Set
+
+import pytest
+
+import datahub.metadata.schema_classes as models
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.source.bigquery_v2.bigquery_audit import (
+    BigQueryTableRef,
+    QueryEvent,
+)
+from datahub.ingestion.source.bigquery_v2.bigquery_config import (
+    BigQueryV2Config,
+    GcsLineageProviderConfig,
+)
+from datahub.ingestion.source.bigquery_v2.bigquery_report import BigQueryV2Report
+from datahub.ingestion.source.bigquery_v2.common import BigQueryIdentifierBuilder
+from datahub.ingestion.source.bigquery_v2.lineage import (
+    BigqueryLineageExtractor,
+    LineageEdge,
+)
+from datahub.ingestion.source.data_lake_common.path_spec import PathSpec
+from datahub.sql_parsing.schema_resolver import SchemaResolver
+
+
+@pytest.fixture
+def lineage_entries() -> List[QueryEvent]:
+    return [
+        QueryEvent(
+            timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
+            actor_email="bla@bla.com",
+            query="""
+                INSERT INTO `my_project.my_dataset.my_table`
+                SELECT first.a, second.b FROM `my_project.my_dataset.my_source_table1` first
+                LEFT JOIN `my_project.my_dataset.my_source_table2` second ON first.id = second.id
+            """,
+            statementType="INSERT",
+            project_id="proj_12344",
+            end_time=None,
+            referencedTables=[
+                BigQueryTableRef.from_string_name(
+                    "projects/my_project/datasets/my_dataset/tables/my_source_table1"
+                ),
+                BigQueryTableRef.from_string_name(
+                    "projects/my_project/datasets/my_dataset/tables/my_source_table2"
+                ),
+            ],
+            destinationTable=BigQueryTableRef.from_string_name(
+                "projects/my_project/datasets/my_dataset/tables/my_table"
+            ),
+        ),
+        QueryEvent(
+            timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
+            actor_email="bla@bla.com",
+            query="testQuery",
+            statementType="SELECT",
+            project_id="proj_12344",
+            end_time=datetime.datetime.fromtimestamp(
+                1617295943.17321, tz=datetime.timezone.utc
+            ),
+            referencedTables=[
+                BigQueryTableRef.from_string_name(
+                    "projects/my_project/datasets/my_dataset/tables/my_source_table3"
+                ),
+            ],
+            destinationTable=BigQueryTableRef.from_string_name(
+                "projects/my_project/datasets/my_dataset/tables/my_table"
+            ),
+        ),
+        QueryEvent(
+            timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
+            actor_email="bla@bla.com",
+            query="testQuery",
+            statementType="SELECT",
+            project_id="proj_12344",
+            referencedViews=[
+                BigQueryTableRef.from_string_name(
+                    "projects/my_project/datasets/my_dataset/tables/my_source_view1"
+                ),
+            ],
+            destinationTable=BigQueryTableRef.from_string_name(
+                "projects/my_project/datasets/my_dataset/tables/my_table"
+            ),
+        ),
+    ]
+
+
+def test_lineage_with_timestamps(lineage_entries: List[QueryEvent]) -> None:
+    config = BigQueryV2Config()
+    report = BigQueryV2Report()
+    extractor: BigqueryLineageExtractor = BigqueryLineageExtractor(
+        config,
+        report,
+        schema_resolver=SchemaResolver(platform="bigquery"),
+        identifiers=BigQueryIdentifierBuilder(config, report),
+    )
+
+    bq_table = BigQueryTableRef.from_string_name(
+        "projects/my_project/datasets/my_dataset/tables/my_table"
+    )
+
+    lineage_map: Dict[str, Set[LineageEdge]] = extractor._create_lineage_map(
+        iter(lineage_entries)
+    )
+
+    upstream_lineage = extractor.get_lineage_for_table(
+        bq_table=bq_table,
+        bq_table_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)",
+        lineage_metadata=lineage_map,
+    )
+    assert upstream_lineage
+    assert len(upstream_lineage.upstreams) == 4
+
+
+def test_column_level_lineage(lineage_entries: List[QueryEvent]) -> None:
+    config = BigQueryV2Config(extract_column_lineage=True, incremental_lineage=False)
+    report = BigQueryV2Report()
+    extractor: BigqueryLineageExtractor = BigqueryLineageExtractor(
+        config,
+        report,
+        schema_resolver=SchemaResolver(platform="bigquery"),
+        identifiers=BigQueryIdentifierBuilder(config, report),
+    )
+
+    bq_table = BigQueryTableRef.from_string_name(
+        "projects/my_project/datasets/my_dataset/tables/my_table"
+    )
+
+    lineage_map: Dict[str, Set[LineageEdge]] = extractor._create_lineage_map(
+        lineage_entries[:1],
+    )
+
+    upstream_lineage = extractor.get_lineage_for_table(
+        bq_table=bq_table,
+        bq_table_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)",
+        lineage_metadata=lineage_map,
+    )
+    assert upstream_lineage
+    assert len(upstream_lineage.upstreams) == 2
+    assert (
+        upstream_lineage.fineGrainedLineages
+        and len(upstream_lineage.fineGrainedLineages) == 2
+    )
+
+
+def test_lineage_for_external_bq_table(mock_datahub_graph_instance):
+    pipeline_context = PipelineContext(run_id="bq_gcs_lineage")
+    pipeline_context.graph = mock_datahub_graph_instance
+
+    def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
+        return models.SchemaMetadataClass(
+            schemaName="sample_schema",
+            platform="urn:li:dataPlatform:gcs",  # important <- platform must be an urn
+            version=0,
+            hash="",
+            platformSchema=models.OtherSchemaClass(
+                rawSchema="__insert raw schema here__"
+            ),
+            fields=[
+                models.SchemaFieldClass(
+                    fieldPath="age",
+                    type=models.SchemaFieldDataTypeClass(type=models.NumberTypeClass()),
+                    nativeDataType="int",
+                ),
+                models.SchemaFieldClass(
+                    fieldPath="firstname",
+                    type=models.SchemaFieldDataTypeClass(type=models.StringTypeClass()),
+                    nativeDataType="VARCHAR(100)",
+                ),
+                models.SchemaFieldClass(
+                    fieldPath="lastname",
+                    type=models.SchemaFieldDataTypeClass(type=models.StringTypeClass()),
+                    nativeDataType="VARCHAR(100)",
+                ),
+            ],
+        )
+
+    pipeline_context.graph.get_schema_metadata = fake_schema_metadata  # type: ignore
+    path_specs: List[PathSpec] = [
+        PathSpec(include="gs://bigquery_data/{table}/*.parquet"),
+        PathSpec(include="gs://bigquery_data/customer3/{table}/*.parquet"),
+    ]
+    gcs_lineage_config: GcsLineageProviderConfig = GcsLineageProviderConfig(
+        path_specs=path_specs
+    )
+
+    config = BigQueryV2Config(
+        include_table_lineage=True,
+        include_column_lineage_with_gcs=True,
+        gcs_lineage_config=gcs_lineage_config,
+    )
+
+    report = BigQueryV2Report()
+    extractor: BigqueryLineageExtractor = BigqueryLineageExtractor(
+        config,
+        report,
+        schema_resolver=SchemaResolver(platform="bigquery"),
+        identifiers=BigQueryIdentifierBuilder(config, report),
+    )
+
+    upstream_lineage = extractor.get_lineage_for_external_table(
+        dataset_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)",
+        source_uris=[
+            "gs://bigquery_data/customer1/*.parquet",
+            "gs://bigquery_data/customer2/*.parquet",
+            "gs://bigquery_data/customer3/my_table/*.parquet",
+        ],
+        graph=pipeline_context.graph,
+    )
+
+    expected_schema_field_urns = [
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer1,PROD),age)",
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer1,PROD),firstname)",
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer1,PROD),lastname)",
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer2,PROD),age)",
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer2,PROD),firstname)",
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer2,PROD),lastname)",
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer3/my_table,PROD),age)",
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer3/my_table,PROD),firstname)",
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer3/my_table,PROD),lastname)",
+    ]
+    assert upstream_lineage
+    assert len(upstream_lineage.upstreams) == 3
+    assert (
+        upstream_lineage.fineGrainedLineages
+        and len(upstream_lineage.fineGrainedLineages) == 9
+    )
+    # Extracting column URNs from upstream_lineage.upstreams
+    actual_schema_field_urns = [
+        fine_grained_lineage.upstreams[0]
+        if fine_grained_lineage.upstreams is not None
+        else []
+        for fine_grained_lineage in upstream_lineage.fineGrainedLineages
+    ]
+    assert all(
+        urn in expected_schema_field_urns for urn in actual_schema_field_urns
+    ), "Some expected column URNs are missing from fine grained lineage."
+
+
+def test_lineage_for_external_bq_table_no_column_lineage(mock_datahub_graph_instance):
+    pipeline_context = PipelineContext(run_id="bq_gcs_lineage")
+    pipeline_context.graph = mock_datahub_graph_instance
+
+    def fake_schema_metadata(entity_urn: str) -> Optional[models.SchemaMetadataClass]:
+        return None
+
+    pipeline_context.graph.get_schema_metadata = fake_schema_metadata  # type: ignore
+    path_specs: List[PathSpec] = [
+        PathSpec(include="gs://bigquery_data/{table}/*.parquet"),
+        PathSpec(include="gs://bigquery_data/customer3/{table}/*.parquet"),
+    ]
+    gcs_lineage_config: GcsLineageProviderConfig = GcsLineageProviderConfig(
+        path_specs=path_specs
+    )
+
+    config = BigQueryV2Config(
+        include_table_lineage=True,
+        include_column_lineage_with_gcs=True,
+        gcs_lineage_config=gcs_lineage_config,
+    )
+
+    report = BigQueryV2Report()
+    extractor: BigqueryLineageExtractor = BigqueryLineageExtractor(
+        config,
+        report,
+        schema_resolver=SchemaResolver(platform="bigquery"),
+        identifiers=BigQueryIdentifierBuilder(config, report),
+    )
+
+    upstream_lineage = extractor.get_lineage_for_external_table(
+        dataset_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)",
+        source_uris=[
+            "gs://bigquery_data/customer1/*.parquet",
+            "gs://bigquery_data/customer2/*.parquet",
+            "gs://bigquery_data/customer3/my_table/*.parquet",
+        ],
+        graph=pipeline_context.graph,
+    )
+
+    expected_dataset_urns = [
+        "urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer1,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer2,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:gcs,bigquery_data/customer3/my_table,PROD)",
+    ]
+    assert upstream_lineage
+    assert len(upstream_lineage.upstreams) == 3
+    # Extracting dataset URNs from upstream_lineage.upstreams
+    actual_dataset_urns = [upstream.dataset for upstream in upstream_lineage.upstreams]
+    assert all(
+        urn in actual_dataset_urns for urn in expected_dataset_urns
+    ), "Some expected dataset URNs are missing from upstream lineage."
+    assert upstream_lineage.fineGrainedLineages is None
+
+
+def test_lineage_for_external_table_with_non_gcs_uri(mock_datahub_graph_instance):
+    pipeline_context = PipelineContext(run_id="non_gcs_lineage")
+    pipeline_context.graph = mock_datahub_graph_instance
+
+    config = BigQueryV2Config(
+        include_table_lineage=True,
+        include_column_lineage_with_gcs=False,  # Column lineage disabled for simplicity
+    )
+    report = BigQueryV2Report()
+    extractor: BigqueryLineageExtractor = BigqueryLineageExtractor(
+        config,
+        report,
+        schema_resolver=SchemaResolver(platform="bigquery"),
+        identifiers=BigQueryIdentifierBuilder(config, report),
+    )
+
+    upstream_lineage = extractor.get_lineage_for_external_table(
+        dataset_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)",
+        source_uris=[
+            "https://some_non_gcs_path/customer1/file.csv",
+            "https://another_path/file.txt",
+        ],
+        graph=pipeline_context.graph,
+    )
+
+    assert upstream_lineage is None
+
+
+def test_lineage_for_external_table_path_not_matching_specs(
+    mock_datahub_graph_instance,
+):
+    pipeline_context = PipelineContext(run_id="path_not_matching_lineage")
+    pipeline_context.graph = mock_datahub_graph_instance
+
+    path_specs: List[PathSpec] = [
+        PathSpec(include="gs://different_data/db2/db3/{table}/*.parquet"),
+    ]
+    gcs_lineage_config: GcsLineageProviderConfig = GcsLineageProviderConfig(
+        path_specs=path_specs, ignore_non_path_spec_path=True
+    )
+    config = BigQueryV2Config(
+        include_table_lineage=True,
+        include_column_lineage_with_gcs=False,
+        gcs_lineage_config=gcs_lineage_config,
+    )
+
+    report = BigQueryV2Report()
+    extractor: BigqueryLineageExtractor = BigqueryLineageExtractor(
+        config,
+        report,
+        schema_resolver=SchemaResolver(platform="bigquery"),
+        identifiers=BigQueryIdentifierBuilder(config, report),
+    )
+
+    upstream_lineage = extractor.get_lineage_for_external_table(
+        dataset_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)",
+        source_uris=[
+            "gs://bigquery_data/customer1/*.parquet",
+            "gs://bigquery_data/customer2/*.parquet",
+        ],
+        graph=pipeline_context.graph,
+    )
+
+    assert upstream_lineage is None
diff --git a/metadata-ingestion/tests/unit/test_bigquery_profiler.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_profiler.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_bigquery_profiler.py
rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_profiler.py
diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py
similarity index 95%
rename from metadata-ingestion/tests/unit/test_bigquery_source.py
rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py
index 38239d150dd6b..b605e9b3f8a3e 100644
--- a/metadata-ingestion/tests/unit/test_bigquery_source.py
+++ b/metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py
@@ -489,30 +489,45 @@ def test_gen_table_dataset_workunits(
     gen = schema_gen.gen_table_dataset_workunits(
         bigquery_table, [], project_id, dataset_name
     )
-    mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata)
-    assert mcp.aspect == StatusClass(removed=False)
+    mcps = list(gen)
+
+    # Helper function to find MCP by aspect type
+    def find_mcp_by_aspect(aspect_type):
+        return next(
+            mcp  # type: ignore
+            for mcp in mcps
+            if isinstance(mcp.metadata.aspect, aspect_type)  # type: ignore
+        )
 
-    mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata)
-    assert isinstance(mcp.aspect, SchemaMetadataClass)
-    assert mcp.aspect.schemaName == f"{project_id}.{dataset_name}.{bigquery_table.name}"
-    assert mcp.aspect.fields == []
+    # Assert StatusClass
+    status_mcp = find_mcp_by_aspect(StatusClass)
+    assert status_mcp.metadata.aspect.removed is False
 
-    mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata)
-    assert isinstance(mcp.aspect, DatasetPropertiesClass)
-    assert mcp.aspect.name == bigquery_table.name
+    # Assert SchemaMetadataClass
+    schema_mcp = find_mcp_by_aspect(SchemaMetadataClass)
+    assert (
+        schema_mcp.metadata.aspect.schemaName
+        == f"{project_id}.{dataset_name}.{bigquery_table.name}"
+    )
+    assert schema_mcp.metadata.aspect.fields == []
+
+    # Assert DatasetPropertiesClass
+    dataset_props_mcp = find_mcp_by_aspect(DatasetPropertiesClass)
+    assert dataset_props_mcp.metadata.aspect.name == bigquery_table.name
     assert (
-        mcp.aspect.qualifiedName == f"{project_id}.{dataset_name}.{bigquery_table.name}"
+        dataset_props_mcp.metadata.aspect.qualifiedName
+        == f"{project_id}.{dataset_name}.{bigquery_table.name}"
     )
-    assert mcp.aspect.description == bigquery_table.comment
-    assert mcp.aspect.created == TimeStampClass(
+    assert dataset_props_mcp.metadata.aspect.description == bigquery_table.comment
+    assert dataset_props_mcp.metadata.aspect.created == TimeStampClass(
         time=int(bigquery_table.created.timestamp() * 1000)
     )
-    assert mcp.aspect.lastModified == TimeStampClass(
+    assert dataset_props_mcp.metadata.aspect.lastModified == TimeStampClass(
         time=int(bigquery_table.last_altered.timestamp() * 1000)
     )
-    assert mcp.aspect.tags == []
+    assert dataset_props_mcp.metadata.aspect.tags == []
 
-    assert mcp.aspect.customProperties == {
+    expected_custom_properties = {
         "expiration_date": str(bigquery_table.expires),
         "size_in_bytes": str(bigquery_table.size_in_bytes),
         "billable_bytes_active": str(bigquery_table.active_billable_bytes),
@@ -523,24 +538,33 @@ def test_gen_table_dataset_workunits(
         "max_shard_id": str(bigquery_table.max_shard_id),
         "is_sharded": "True",
     }
+    assert (
+        dataset_props_mcp.metadata.aspect.customProperties == expected_custom_properties
+    )
 
-    mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata)
-    assert isinstance(mcp.aspect, GlobalTagsClass)
-    assert mcp.aspect.tags == [
+    # Assert GlobalTagsClass
+    global_tags_mcp = find_mcp_by_aspect(GlobalTagsClass)
+    assert global_tags_mcp.metadata.aspect.tags == [
         TagAssociationClass(
             "urn:li:tag:data_producer_owner_email:games_team-nytimes_com"
         )
     ]
 
-    mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata)
-    assert isinstance(mcp.aspect, ContainerClass)
+    # Assert ContainerClass
+    container_mcp = find_mcp_by_aspect(ContainerClass)
+    assert container_mcp is not None
 
-    mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata)
-    assert isinstance(mcp.aspect, DataPlatformInstanceClass)
+    # Assert DataPlatformInstanceClass
+    data_platform_instance_mcp = find_mcp_by_aspect(DataPlatformInstanceClass)
+    assert data_platform_instance_mcp is not None
 
-    mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata)
-    assert isinstance(mcp.aspect, SubTypesClass)
-    assert mcp.aspect.typeNames[1] == DatasetSubTypes.TABLE
+    # Assert SubTypesClass
+    sub_types_mcp = find_mcp_by_aspect(SubTypesClass)
+    assert sub_types_mcp.metadata.aspect.typeNames[1] == DatasetSubTypes.TABLE
+
+    # Ensure all MCPs were checked
+    # TODO: Test for PlatformResource MCPs as well
+    assert len(mcps) >= 7
 
 
 @patch.object(BigQueryV2Config, "get_bigquery_client")
diff --git a/metadata-ingestion/tests/unit/test_bigquery_sql_lineage.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_sql_lineage.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_bigquery_sql_lineage.py
rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_sql_lineage.py
diff --git a/metadata-ingestion/tests/unit/test_bigquery_usage.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_usage.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_bigquery_usage.py
rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_usage.py
diff --git a/metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py b/metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py
similarity index 99%
rename from metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py
rename to metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py
index 63de742b201a9..3247a64631da7 100644
--- a/metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py
+++ b/metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py
@@ -184,7 +184,6 @@ def test_bigquery_table_sanitasitation():
 
 
 def test_unquote_and_decode_unicode_escape_seq():
-
     # Test with a string that starts and ends with quotes and has Unicode escape sequences
     input_string = '"Hello \\u003cWorld\\u003e"'
     expected_output = "Hello <World>"
diff --git a/metadata-ingestion/tests/unit/test_bq_get_partition_range.py b/metadata-ingestion/tests/unit/bigquery/test_bq_get_partition_range.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_bq_get_partition_range.py
rename to metadata-ingestion/tests/unit/bigquery/test_bq_get_partition_range.py
diff --git a/metadata-ingestion/tests/unit/test_check.py b/metadata-ingestion/tests/unit/cli/test_check.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_check.py
rename to metadata-ingestion/tests/unit/cli/test_check.py
diff --git a/metadata-ingestion/tests/unit/test_check_upgrade.py b/metadata-ingestion/tests/unit/cli/test_check_upgrade.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_check_upgrade.py
rename to metadata-ingestion/tests/unit/cli/test_check_upgrade.py
diff --git a/metadata-ingestion/tests/unit/test_cli_utils.py b/metadata-ingestion/tests/unit/cli/test_cli_utils.py
similarity index 95%
rename from metadata-ingestion/tests/unit/test_cli_utils.py
rename to metadata-ingestion/tests/unit/cli/test_cli_utils.py
index af3a184d97e41..c9693c75d96fe 100644
--- a/metadata-ingestion/tests/unit/test_cli_utils.py
+++ b/metadata-ingestion/tests/unit/cli/test_cli_utils.py
@@ -66,6 +66,10 @@ def test_fixup_gms_url():
     assert cli_utils.fixup_gms_url("http://localhost:8080") == "http://localhost:8080"
     assert cli_utils.fixup_gms_url("http://localhost:8080/") == "http://localhost:8080"
     assert cli_utils.fixup_gms_url("http://abc.acryl.io") == "https://abc.acryl.io/gms"
+    assert (
+        cli_utils.fixup_gms_url("http://abc.acryl.io/api/gms")
+        == "https://abc.acryl.io/gms"
+    )
 
 
 def test_guess_frontend_url_from_gms_url():
diff --git a/metadata-ingestion/tests/unit/test_key_value_pattern.py b/metadata-ingestion/tests/unit/config/test_key_value_pattern.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_key_value_pattern.py
rename to metadata-ingestion/tests/unit/config/test_key_value_pattern.py
diff --git a/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py b/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py
new file mode 100644
index 0000000000000..27df05296b752
--- /dev/null
+++ b/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py
@@ -0,0 +1,123 @@
+from unittest.mock import Mock
+
+import pytest
+
+from datahub.ingestion.source.dremio.dremio_api import DremioAPIOperations
+from datahub.ingestion.source.dremio.dremio_config import DremioSourceConfig
+from datahub.ingestion.source.dremio.dremio_reporting import DremioSourceReport
+
+
+class TestDremioContainerFiltering:
+    @pytest.fixture
+    def dremio_api(self, monkeypatch):
+        # Mock the requests.Session
+        mock_session = Mock()
+        monkeypatch.setattr("requests.Session", Mock(return_value=mock_session))
+
+        # Mock the authentication response
+        mock_session.post.return_value.json.return_value = {"token": "dummy-token"}
+        mock_session.post.return_value.status_code = 200
+
+        config = DremioSourceConfig(
+            hostname="dummy-host",
+            port=9047,
+            tls=False,
+            authentication_method="password",
+            username="dummy-user",
+            password="dummy-password",
+            schema_pattern=dict(allow=[".*"], deny=[]),
+        )
+        report = DremioSourceReport()
+        return DremioAPIOperations(config, report)
+
+    def test_basic_allow_pattern(self, dremio_api):
+        """Test basic allow pattern matching"""
+        dremio_api.allow_schema_pattern = ["test"]
+        dremio_api.deny_schema_pattern = []
+
+        assert dremio_api.should_include_container([], "test")
+        assert dremio_api.should_include_container(["test"], "subfolder")
+        assert not dremio_api.should_include_container([], "prod_space")
+
+    def test_basic_deny_pattern(self, dremio_api):
+        """Test basic deny pattern matching"""
+        dremio_api.allow_schema_pattern = [".*"]
+        dremio_api.deny_schema_pattern = ["test_space.*"]
+
+        assert not dremio_api.should_include_container([], "test_space")
+        assert not dremio_api.should_include_container(["test_space"], "subfolder")
+        assert dremio_api.should_include_container([], "prod_space")
+
+    def test_hierarchical_matching(self, dremio_api):
+        """Test matching with hierarchical paths"""
+        dremio_api.allow_schema_pattern = ["prod.data.*"]
+        dremio_api.deny_schema_pattern = []
+
+        assert dremio_api.should_include_container([], "prod")
+        assert dremio_api.should_include_container(["prod"], "data")
+        assert dremio_api.should_include_container(["prod", "data"], "sales")
+        assert not dremio_api.should_include_container([], "dev")
+        assert not dremio_api.should_include_container(["dev"], "data")
+
+    def test_allow_and_deny_patterns(self, dremio_api):
+        """Test combination of allow and deny patterns"""
+        dremio_api.allow_schema_pattern = ["prod.*"]
+        dremio_api.deny_schema_pattern = ["prod.internal.*"]
+
+        assert dremio_api.should_include_container([], "prod")
+        assert dremio_api.should_include_container(["prod"], "public")
+        assert dremio_api.should_include_container(["prod", "public"], "next")
+        assert not dremio_api.should_include_container(["prod"], "internal")
+        assert not dremio_api.should_include_container(["prod", "internal"], "secrets")
+
+    def test_wildcard_patterns(self, dremio_api):
+        """Test wildcard pattern handling"""
+        dremio_api.allow_schema_pattern = [".*"]
+        dremio_api.deny_schema_pattern = []
+
+        assert dremio_api.should_include_container([], "any_space")
+        assert dremio_api.should_include_container(["any_space"], "any_folder")
+
+        # Test with specific wildcard in middle
+        dremio_api.allow_schema_pattern = ["prod.*.public"]
+        assert dremio_api.should_include_container(["prod", "customer"], "public")
+        assert not dremio_api.should_include_container(["prod", "customer"], "private")
+
+    def test_case_insensitive_matching(self, dremio_api):
+        """Test case-insensitive pattern matching"""
+        dremio_api.allow_schema_pattern = ["PROD.*"]
+        dremio_api.deny_schema_pattern = []
+
+        assert dremio_api.should_include_container([], "prod")
+        assert dremio_api.should_include_container([], "PROD")
+        assert dremio_api.should_include_container(["prod"], "DATA")
+        assert dremio_api.should_include_container(["PROD"], "data")
+
+    def test_empty_patterns(self, dremio_api):
+        """Test behavior with empty patterns"""
+        dremio_api.allow_schema_pattern = [".*"]
+        dremio_api.deny_schema_pattern = []
+
+        # Should allow everything when allow pattern is empty
+        assert dremio_api.should_include_container([], "any_space")
+        assert dremio_api.should_include_container(["any_space"], "any_folder")
+
+    def test_partial_path_matching(self, dremio_api):
+        """Test matching behavior with partial paths"""
+        dremio_api.allow_schema_pattern = ["^pr.*.data.*"]
+        dremio_api.deny_schema_pattern = []
+
+        assert dremio_api.should_include_container(["prod"], "data")
+        # Should match the partial path even though pattern doesn't have wildcards
+        assert dremio_api.should_include_container(["prod", "data"], "sales")
+        assert not dremio_api.should_include_container([], "dev")
+        assert not dremio_api.should_include_container(["dev", "data"], "sales")
+
+    def test_partial_start_end_chars(self, dremio_api):
+        """Test matching behavior with partial paths"""
+        dremio_api.allow_schema_pattern = ["pr.*.data$"]
+        dremio_api.deny_schema_pattern = []
+
+        assert dremio_api.should_include_container(["prod"], "data")
+        # Should match the partial path even though pattern doesn't have wildcards
+        assert not dremio_api.should_include_container(["prod", "data"], "sales")
diff --git a/metadata-ingestion/tests/unit/glue/__init__.py b/metadata-ingestion/tests/unit/glue/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/glue/test_glue_source.py
similarity index 94%
rename from metadata-ingestion/tests/unit/test_glue_source.py
rename to metadata-ingestion/tests/unit/glue/test_glue_source.py
index 45b9899eacaa7..693fd6bc336fd 100644
--- a/metadata-ingestion/tests/unit/test_glue_source.py
+++ b/metadata-ingestion/tests/unit/glue/test_glue_source.py
@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from typing import Any, Callable, Dict, Optional, Tuple, Type, cast
+from typing import Any, Dict, Optional, Tuple, Type, cast
 from unittest.mock import patch
 
 import pydantic
@@ -11,7 +11,7 @@
 import datahub.metadata.schema_classes as models
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields
-from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
+from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.sink.file import write_metadata_file
 from datahub.ingestion.source.aws.glue import (
     GlueProfilingConfig,
@@ -34,8 +34,7 @@
     run_and_get_pipeline,
     validate_all_providers_have_committed_successfully,
 )
-from tests.test_helpers.type_helpers import PytestConfig
-from tests.unit.test_glue_source_stubs import (
+from tests.unit.glue.test_glue_source_stubs import (
     databases_1,
     databases_2,
     get_bucket_tagging,
@@ -71,10 +70,12 @@
 GMS_PORT = 8080
 GMS_SERVER = f"http://localhost:{GMS_PORT}"
 
+test_resources_dir = Path(__file__).parent
+
 
 def glue_source(
     platform_instance: Optional[str] = None,
-    mock_datahub_graph: Optional[Callable[[DatahubClientConfig], DataHubGraph]] = None,
+    mock_datahub_graph_instance: Optional[DataHubGraph] = None,
     use_s3_bucket_tags: bool = True,
     use_s3_object_tags: bool = True,
     extract_delta_schema_from_parameters: bool = False,
@@ -83,8 +84,8 @@ def glue_source(
     extract_transforms: bool = True,
 ) -> GlueSource:
     pipeline_context = PipelineContext(run_id="glue-source-tes")
-    if mock_datahub_graph:
-        pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    if mock_datahub_graph_instance:
+        pipeline_context.graph = mock_datahub_graph_instance
     return GlueSource(
         ctx=pipeline_context,
         config=GlueSourceConfig(
@@ -172,7 +173,7 @@ def test_column_type(hive_column_type: str, expected_type: Type) -> None:
 @freeze_time(FROZEN_TIME)
 def test_glue_ingest(
     tmp_path: Path,
-    pytestconfig: PytestConfig,
+    pytestconfig: pytest.Config,
     platform_instance: str,
     mce_file: str,
     mce_golden_file: str,
@@ -247,7 +248,6 @@ def test_glue_ingest(
             write_metadata_file(tmp_path / mce_file, mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / mce_file,
@@ -266,8 +266,8 @@ def test_platform_config():
 @pytest.mark.parametrize(
     "ignore_resource_links, all_databases_and_tables_result",
     [
-        (True, ({}, [])),
-        (False, ({"test-database": resource_link_database}, target_database_tables)),
+        (True, ([], [])),
+        (False, ([resource_link_database], target_database_tables)),
     ],
 )
 def test_ignore_resource_links(ignore_resource_links, all_databases_and_tables_result):
@@ -288,7 +288,7 @@ def test_ignore_resource_links(ignore_resource_links, all_databases_and_tables_r
         glue_stubber.add_response(
             "get_tables",
             get_tables_response_for_target_database,
-            {"DatabaseName": "test-database"},
+            {"DatabaseName": "resource-link-test-database"},
         )
 
         assert source.get_all_databases_and_tables() == all_databases_and_tables_result
@@ -312,8 +312,6 @@ def test_config_without_platform():
 
 @freeze_time(FROZEN_TIME)
 def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
-
     deleted_actor_golden_mcs = "{}/glue_deleted_actor_mces_golden.json".format(
         test_resources_dir
     )
@@ -411,7 +409,7 @@ def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
 
 def test_glue_with_delta_schema_ingest(
     tmp_path: Path,
-    pytestconfig: PytestConfig,
+    pytestconfig: pytest.Config,
 ) -> None:
     glue_source_instance = glue_source(
         platform_instance="delta_platform_instance",
@@ -438,7 +436,6 @@ def test_glue_with_delta_schema_ingest(
         write_metadata_file(tmp_path / "glue_delta_mces.json", mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / "glue_delta_mces.json",
@@ -448,7 +445,7 @@ def test_glue_with_delta_schema_ingest(
 
 def test_glue_with_malformed_delta_schema_ingest(
     tmp_path: Path,
-    pytestconfig: PytestConfig,
+    pytestconfig: pytest.Config,
 ) -> None:
     glue_source_instance = glue_source(
         platform_instance="delta_platform_instance",
@@ -475,7 +472,6 @@ def test_glue_with_malformed_delta_schema_ingest(
         write_metadata_file(tmp_path / "glue_malformed_delta_mces.json", mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / "glue_malformed_delta_mces.json",
@@ -492,15 +488,15 @@ def test_glue_with_malformed_delta_schema_ingest(
 @freeze_time(FROZEN_TIME)
 def test_glue_ingest_include_table_lineage(
     tmp_path: Path,
-    pytestconfig: PytestConfig,
-    mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph],
+    pytestconfig: pytest.Config,
+    mock_datahub_graph_instance: DataHubGraph,
     platform_instance: str,
     mce_file: str,
     mce_golden_file: str,
 ) -> None:
     glue_source_instance = glue_source(
         platform_instance=platform_instance,
-        mock_datahub_graph=mock_datahub_graph,
+        mock_datahub_graph_instance=mock_datahub_graph_instance,
         emit_s3_lineage=True,
     )
 
@@ -571,7 +567,6 @@ def test_glue_ingest_include_table_lineage(
             write_metadata_file(tmp_path / mce_file, mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / mce_file,
@@ -588,15 +583,15 @@ def test_glue_ingest_include_table_lineage(
 @freeze_time(FROZEN_TIME)
 def test_glue_ingest_include_column_lineage(
     tmp_path: Path,
-    pytestconfig: PytestConfig,
-    mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph],
+    pytestconfig: pytest.Config,
+    mock_datahub_graph_instance: DataHubGraph,
     platform_instance: str,
     mce_file: str,
     mce_golden_file: str,
 ) -> None:
     glue_source_instance = glue_source(
         platform_instance=platform_instance,
-        mock_datahub_graph=mock_datahub_graph,
+        mock_datahub_graph_instance=mock_datahub_graph_instance,
         emit_s3_lineage=True,
         include_column_lineage=True,
         use_s3_bucket_tags=False,
@@ -678,7 +673,6 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
         write_metadata_file(tmp_path / mce_file, mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / mce_file,
@@ -689,7 +683,7 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
 @freeze_time(FROZEN_TIME)
 def test_glue_ingest_with_profiling(
     tmp_path: Path,
-    pytestconfig: PytestConfig,
+    pytestconfig: pytest.Config,
 ) -> None:
     glue_source_instance = glue_source_with_profiling()
     mce_file = "glue_mces.json"
@@ -716,7 +710,6 @@ def test_glue_ingest_with_profiling(
         write_metadata_file(tmp_path / mce_file, mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / mce_file,
diff --git a/metadata-ingestion/tests/unit/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py
similarity index 99%
rename from metadata-ingestion/tests/unit/test_glue_source_stubs.py
rename to metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py
index 46ab65234c22d..dba1eea3010c2 100644
--- a/metadata-ingestion/tests/unit/test_glue_source_stubs.py
+++ b/metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py
@@ -5,7 +5,7 @@
 from botocore.response import StreamingBody
 
 resource_link_database = {
-    "Name": "test-database",
+    "Name": "resource-link-test-database",
     "CreateTime": datetime.datetime(2021, 6, 9, 14, 14, 19),
     "CreateTableDefaultPermissions": [],
     "TargetDatabase": {"CatalogId": "432143214321", "DatabaseName": "test-database"},
@@ -92,10 +92,8 @@
         },
     ]
 }
-databases_1 = {
-    "flights-database": {"Name": "flights-database", "CatalogId": "123412341234"}
-}
-databases_2 = {"test-database": {"Name": "test-database", "CatalogId": "123412341234"}}
+databases_1 = [{"Name": "flights-database", "CatalogId": "123412341234"}]
+databases_2 = [{"Name": "test-database", "CatalogId": "123412341234"}]
 tables_1 = [
     {
         "Name": "avro",
diff --git a/metadata-ingestion/tests/unit/patch/test_patch_builder.py b/metadata-ingestion/tests/unit/patch/test_patch_builder.py
index 267da6cdd5d20..f4bf501e0714d 100644
--- a/metadata-ingestion/tests/unit/patch/test_patch_builder.py
+++ b/metadata-ingestion/tests/unit/patch/test_patch_builder.py
@@ -1,5 +1,6 @@
 import json
 import pathlib
+from typing import Any, Dict, Union
 
 import pytest
 from freezegun.api import freeze_time
@@ -15,7 +16,9 @@
 )
 from datahub.ingestion.sink.file import write_metadata_file
 from datahub.metadata.schema_classes import (
+    AuditStampClass,
     DatasetLineageTypeClass,
+    EdgeClass,
     FineGrainedLineageClass,
     FineGrainedLineageDownstreamTypeClass,
     FineGrainedLineageUpstreamTypeClass,
@@ -182,8 +185,66 @@ def test_basic_dashboard_patch_builder():
     ]
 
 
+@pytest.mark.parametrize(
+    "created_on,last_modified,expected_actor",
+    [
+        (1586847600000, 1586847600000, "urn:li:corpuser:datahub"),
+        (None, None, "urn:li:corpuser:datahub"),
+        (1586847600000, None, "urn:li:corpuser:datahub"),
+        (None, 1586847600000, "urn:li:corpuser:datahub"),
+    ],
+    ids=["both_timestamps", "no_timestamps", "only_created", "only_modified"],
+)
 @freeze_time("2020-04-14 07:00:00")
-def test_datajob_patch_builder():
+def test_datajob_patch_builder(created_on, last_modified, expected_actor):
+    def make_edge_or_urn(urn: str) -> Union[EdgeClass, str]:
+        if created_on or last_modified:
+            return EdgeClass(
+                destinationUrn=str(urn),
+                created=(
+                    AuditStampClass(
+                        time=created_on,
+                        actor=expected_actor,
+                    )
+                    if created_on
+                    else None
+                ),
+                lastModified=(
+                    AuditStampClass(
+                        time=last_modified,
+                        actor=expected_actor,
+                    )
+                    if last_modified
+                    else None
+                ),
+            )
+        return urn
+
+    def get_edge_expectation(urn: str) -> Dict[str, Any]:
+        if created_on or last_modified:
+            expected = {
+                "destinationUrn": str(urn),
+                "created": (
+                    AuditStampClass(
+                        time=created_on,
+                        actor=expected_actor,
+                    ).to_obj()
+                    if created_on
+                    else None
+                ),
+                "lastModified": (
+                    AuditStampClass(
+                        time=last_modified,
+                        actor=expected_actor,
+                    ).to_obj()
+                    if last_modified
+                    else None
+                ),
+            }
+            # filter out None values
+            return {k: v for k, v in expected.items() if v is not None}
+        return {"destinationUrn": str(urn)}
+
     flow_urn = make_data_flow_urn(
         orchestrator="nifi", flow_id="252C34e5af19-0192-1000-b248-b1abee565b5d"
     )
@@ -193,13 +254,19 @@ def test_datajob_patch_builder():
     patcher = DataJobPatchBuilder(job_urn)
 
     patcher.add_output_dataset(
-        "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder1,DEV)"
+        make_edge_or_urn(
+            "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder1,DEV)"
+        )
     )
     patcher.add_output_dataset(
-        "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder3,DEV)"
+        make_edge_or_urn(
+            "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder3,DEV)"
+        )
     )
     patcher.add_output_dataset(
-        "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder2,DEV)"
+        make_edge_or_urn(
+            "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder2,DEV)"
+        )
     )
 
     assert patcher.build() == [
@@ -214,47 +281,23 @@ def test_datajob_patch_builder():
                         {
                             "op": "add",
                             "path": "/outputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket~1folder1,DEV)",
-                            "value": {
-                                "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder1,DEV)",
-                                "created": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub",
-                                },
-                                "lastModified": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub",
-                                },
-                            },
+                            "value": get_edge_expectation(
+                                "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder1,DEV)"
+                            ),
                         },
                         {
                             "op": "add",
                             "path": "/outputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket~1folder3,DEV)",
-                            "value": {
-                                "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder3,DEV)",
-                                "created": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub",
-                                },
-                                "lastModified": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub",
-                                },
-                            },
+                            "value": get_edge_expectation(
+                                "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder3,DEV)"
+                            ),
                         },
                         {
                             "op": "add",
                             "path": "/outputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket~1folder2,DEV)",
-                            "value": {
-                                "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder2,DEV)",
-                                "created": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub",
-                                },
-                                "lastModified": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub",
-                                },
-                            },
+                            "value": get_edge_expectation(
+                                "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder2,DEV)"
+                            ),
                         },
                     ]
                 ).encode("utf-8"),
diff --git a/metadata-ingestion/tests/unit/redshift/__init__.py b/metadata-ingestion/tests/unit/redshift/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py b/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py
new file mode 100644
index 0000000000000..10c0250e37e37
--- /dev/null
+++ b/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py
@@ -0,0 +1,66 @@
+from datetime import datetime
+from unittest.mock import MagicMock
+
+
+def mock_temp_table_cursor(cursor: MagicMock) -> None:
+    cursor.description = [
+        ["transaction_id"],
+        ["session_id"],
+        ["query_text"],
+        ["create_command"],
+        ["start_time"],
+    ]
+
+    cursor.fetchmany.side_effect = [
+        [
+            (
+                126,
+                "abc",
+                "CREATE TABLE #player_price distkey(player_id) AS SELECT player_id, SUM(price) AS "
+                "price_usd from player_activity group by player_id",
+                "CREATE TABLE #player_price",
+                datetime.now(),
+            )
+        ],
+        [
+            # Empty result to stop the while loop
+        ],
+    ]
+
+
+def mock_stl_insert_table_cursor(cursor: MagicMock) -> None:
+    cursor.description = [
+        ["source_schema"],
+        ["source_table"],
+        ["target_schema"],
+        ["target_table"],
+        ["ddl"],
+    ]
+
+    cursor.fetchmany.side_effect = [
+        [
+            (
+                "public",
+                "#player_price",
+                "public",
+                "player_price_with_hike_v6",
+                "INSERT INTO player_price_with_hike_v6 SELECT (price_usd + 0.2 * price_usd) as price, '20%' FROM "
+                "#player_price",
+            )
+        ],
+        [
+            # Empty result to stop the while loop
+        ],
+    ]
+
+
+query_vs_cursor_mocker = {
+    (
+        "-- DataHub Redshift Source temp table DDL query\nselect\n    *\nfrom (\n    select\n        session_id,\n        transaction_id,\n        start_time,\n        userid,\n        REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\\\\\n','\\\\n'), '(CREATE(?:[\\\\n\\\\s\\\\t]+(?:temp|temporary))?(?:[\\\\n\\\\s\\\\t]+)table(?:[\\\\n\\\\s\\\\t]+)[^\\\\n\\\\s\\\\t()-]+)', 0, 1, 'ipe'),'[\\\\n\\\\s\\\\t]+',' ',1,'p') as create_command,\n        query_text,\n        row_number() over (\n            partition by session_id, TRIM(query_text)\n            order by start_time desc\n        ) rn\n    from (\n        select\n            pid as session_id,\n            xid as transaction_id,\n            starttime as start_time,\n            type,\n            query_text,\n            userid\n        from (\n            select\n                starttime,\n                pid,\n                xid,\n                type,\n                userid,\n                LISTAGG(case\n                    when LEN(RTRIM(text)) = 0 then text\n                    else RTRIM(text)\n                end,\n                '') within group (\n                    order by sequence\n                ) as query_text\n            from\n                SVL_STATEMENTTEXT\n            where\n                type in ('DDL', 'QUERY')\n                AND        starttime >= '2024-01-01 12:00:00'\n                AND        starttime < '2024-01-10 12:00:00'\n                AND sequence < 290\n            group by\n                starttime,\n                pid,\n                xid,\n                type,\n                userid\n            order by\n                starttime,\n                pid,\n                xid,\n                type,\n                userid\n                asc\n        )\n        where\n            type in ('DDL', 'QUERY')\n    )\n    where\n        (create_command ilike 'create temp table %'\n            or create_command ilike 'create temporary table %'\n            -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction\n            or create_command ilike 'create table %')\n        -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out.\n        and query_text not ilike 'CREATE TEMP TABLE volt_tt_%'\n        and create_command not like 'CREATE TEMP TABLE volt_tt_'\n        -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query\n        and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%'\n\n)\nwhere\n    rn = 1\n"
+    ): mock_temp_table_cursor,
+    "select * from test_collapse_temp_lineage": mock_stl_insert_table_cursor,
+}
+
+
+def mock_cursor(cursor: MagicMock, query: str) -> None:
+    query_vs_cursor_mocker[query](cursor=cursor)
diff --git a/metadata-ingestion/tests/unit/test_redshift_config.py b/metadata-ingestion/tests/unit/redshift/test_redshift_config.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_redshift_config.py
rename to metadata-ingestion/tests/unit/redshift/test_redshift_config.py
diff --git a/metadata-ingestion/tests/unit/test_redshift_lineage.py b/metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py
similarity index 99%
rename from metadata-ingestion/tests/unit/test_redshift_lineage.py
rename to metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py
index 78b7169a93f3c..941d13be0a613 100644
--- a/metadata-ingestion/tests/unit/test_redshift_lineage.py
+++ b/metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py
@@ -26,7 +26,7 @@
     SqlParsingDebugInfo,
     SqlParsingResult,
 )
-from tests.unit.redshift_query_mocker import mock_cursor
+from tests.unit.redshift.redshift_query_mocker import mock_cursor
 
 
 def test_get_sources_from_query():
@@ -221,7 +221,6 @@ def mock_redshift_connection() -> MagicMock:
 
 
 def mock_graph() -> DataHubGraph:
-
     graph = MagicMock()
 
     graph._make_schema_resolver.return_value = SchemaResolver(
diff --git a/metadata-ingestion/tests/unit/test_redshift_source.py b/metadata-ingestion/tests/unit/redshift/test_redshift_source.py
similarity index 93%
rename from metadata-ingestion/tests/unit/test_redshift_source.py
rename to metadata-ingestion/tests/unit/redshift/test_redshift_source.py
index 8198caf50df7f..f016312dfe47f 100644
--- a/metadata-ingestion/tests/unit/test_redshift_source.py
+++ b/metadata-ingestion/tests/unit/redshift/test_redshift_source.py
@@ -1,15 +1,15 @@
 from typing import Iterable
 
-from datahub.emitter.mcp import (
-    MetadataChangeProposalClass,
-    MetadataChangeProposalWrapper,
-)
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.redshift.config import RedshiftConfig
 from datahub.ingestion.source.redshift.redshift import RedshiftSource
 from datahub.ingestion.source.redshift.redshift_schema import RedshiftTable
-from datahub.metadata.schema_classes import MetadataChangeEventClass
+from datahub.metadata.schema_classes import (
+    MetadataChangeEventClass,
+    MetadataChangeProposalClass,
+)
 
 
 def redshift_source_setup(custom_props_flag: bool) -> Iterable[MetadataWorkUnit]:
diff --git a/metadata-ingestion/tests/unit/redshift_query_mocker.py b/metadata-ingestion/tests/unit/redshift_query_mocker.py
deleted file mode 100644
index ada76e624032b..0000000000000
--- a/metadata-ingestion/tests/unit/redshift_query_mocker.py
+++ /dev/null
@@ -1,104 +0,0 @@
-from datetime import datetime
-from unittest.mock import MagicMock
-
-
-def mock_temp_table_cursor(cursor: MagicMock) -> None:
-    cursor.description = [
-        ["transaction_id"],
-        ["session_id"],
-        ["query_text"],
-        ["create_command"],
-        ["start_time"],
-    ]
-
-    cursor.fetchmany.side_effect = [
-        [
-            (
-                126,
-                "abc",
-                "CREATE TABLE #player_price distkey(player_id) AS SELECT player_id, SUM(price) AS "
-                "price_usd from player_activity group by player_id",
-                "CREATE TABLE #player_price",
-                datetime.now(),
-            )
-        ],
-        [
-            # Empty result to stop the while loop
-        ],
-    ]
-
-
-def mock_stl_insert_table_cursor(cursor: MagicMock) -> None:
-    cursor.description = [
-        ["source_schema"],
-        ["source_table"],
-        ["target_schema"],
-        ["target_table"],
-        ["ddl"],
-    ]
-
-    cursor.fetchmany.side_effect = [
-        [
-            (
-                "public",
-                "#player_price",
-                "public",
-                "player_price_with_hike_v6",
-                "INSERT INTO player_price_with_hike_v6 SELECT (price_usd + 0.2 * price_usd) as price, '20%' FROM "
-                "#player_price",
-            )
-        ],
-        [
-            # Empty result to stop the while loop
-        ],
-    ]
-
-
-query_vs_cursor_mocker = {
-    (
-        "-- DataHub Redshift Source temp table DDL query\n            select\n                *\n            "
-        "from\n                (\n                select\n                    session_id,\n                  "
-        "  transaction_id,\n                    start_time,\n                    userid,\n                   "
-        " REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\\\\\n','\\\\n'), '(CREATE(?:["
-        "\\\\n\\\\s\\\\t]+(?:temp|temporary))?(?:[\\\\n\\\\s\\\\t]+)table(?:[\\\\n\\\\s\\\\t]+)["
-        "^\\\\n\\\\s\\\\t()-]+)', 0, 1, 'ipe'),'[\\\\n\\\\s\\\\t]+',' ',1,'p') as create_command,\n          "
-        "          query_text,\n                    row_number() over (\n                        partition "
-        "by session_id, TRIM(query_text)\n                        order by start_time desc\n                    ) rn\n   "
-        "             from\n                    (\n                    select\n                        pid "
-        "as session_id,\n                        xid as transaction_id,\n                        starttime "
-        "as start_time,\n                        type,\n                        query_text,\n                "
-        "        userid\n                    from\n                        (\n                        "
-        "select\n                            starttime,\n                            pid,\n                  "
-        "          xid,\n                            type,\n                            userid,\n            "
-        "                LISTAGG(case\n                                when LEN(RTRIM(text)) = 0 then text\n "
-        "                               else RTRIM(text)\n                            end,\n                 "
-        "           '') within group (\n                                order by sequence\n                  "
-        "          ) as query_text\n                        from\n                            "
-        "SVL_STATEMENTTEXT\n                        where\n                            type in ('DDL', "
-        "'QUERY')\n                            AND        starttime >= '2024-01-01 12:00:00'\n               "
-        "             AND        starttime < '2024-01-10 12:00:00'\n                            -- See "
-        "https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl"
-        "-statementtext\n                            AND sequence < 320\n                        group by\n  "
-        "                          starttime,\n                            pid,\n                            "
-        "xid,\n                            type,\n                            userid\n                       "
-        " order by\n                            starttime,\n                            pid,\n               "
-        "             xid,\n                            type,\n                            userid\n          "
-        "                  asc)\n                    where\n                        type in ('DDL', "
-        "'QUERY')\n                )\n                where\n                    (create_command ilike "
-        "'create temp table %'\n                        or create_command ilike 'create temporary table %'\n "
-        "                       -- we want to get all the create table statements and not just temp tables "
-        "if non temp table is created and dropped in the same transaction\n                        or "
-        "create_command ilike 'create table %')\n                    -- Redshift creates temp tables with "
-        "the following names: volt_tt_%. We need to filter them out.\n                    and query_text not "
-        "ilike 'CREATE TEMP TABLE volt_tt_%'\n                    and create_command not like 'CREATE TEMP "
-        "TABLE volt_tt_'\n                    -- We need to filter out our query and it was not possible "
-        "earlier when we did not have any comment in the query\n                    and query_text not ilike "
-        "'%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl"
-        "-statementtext%'\n\n            )\n            where\n                rn = 1\n            "
-    ): mock_temp_table_cursor,
-    "select * from test_collapse_temp_lineage": mock_stl_insert_table_cursor,
-}
-
-
-def mock_cursor(cursor: MagicMock, query: str) -> None:
-    query_vs_cursor_mocker[query](cursor=cursor)
diff --git a/metadata-ingestion/tests/unit/sagemaker/__init__.py b/metadata-ingestion/tests/unit/sagemaker/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py
index 995d176c213b2..c7a1fab068a83 100644
--- a/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py
+++ b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py
@@ -3,6 +3,7 @@
 from botocore.stub import Stubber
 from freezegun import freeze_time
 
+import datahub.ingestion.source.aws.sagemaker_processors.models
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.sink.file import write_metadata_file
 from datahub.ingestion.source.aws.sagemaker import (
@@ -13,8 +14,9 @@
     job_type_to_info,
     job_types,
 )
+from datahub.testing.doctest import assert_doctest
 from tests.test_helpers import mce_helpers
-from tests.unit.test_sagemaker_source_stubs import (
+from tests.unit.sagemaker.test_sagemaker_source_stubs import (
     describe_endpoint_response_1,
     describe_endpoint_response_2,
     describe_feature_group_response_1,
@@ -241,3 +243,7 @@ def test_sagemaker_ingest(tmp_path, pytestconfig):
         output_path=tmp_path / "sagemaker_mces.json",
         golden_path=test_resources_dir / "sagemaker_mces_golden.json",
     )
+
+
+def test_doc_test_run() -> None:
+    assert_doctest(datahub.ingestion.source.aws.sagemaker_processors.models)
diff --git a/metadata-ingestion/tests/unit/test_sagemaker_source_stubs.py b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source_stubs.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_sagemaker_source_stubs.py
rename to metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source_stubs.py
diff --git a/metadata-ingestion/tests/unit/graph/test_client.py b/metadata-ingestion/tests/unit/sdk/test_client.py
similarity index 95%
rename from metadata-ingestion/tests/unit/graph/test_client.py
rename to metadata-ingestion/tests/unit/sdk/test_client.py
index faed1f51b29aa..16795ef8c7f81 100644
--- a/metadata-ingestion/tests/unit/graph/test_client.py
+++ b/metadata-ingestion/tests/unit/sdk/test_client.py
@@ -11,7 +11,7 @@
 @patch("datahub.emitter.rest_emitter.DataHubRestEmitter.test_connection")
 def test_get_aspect(mock_test_connection):
     mock_test_connection.return_value = {}
-    graph = DataHubGraph(DatahubClientConfig())
+    graph = DataHubGraph(DatahubClientConfig(server="http://fake-domain.local"))
     user_urn = "urn:li:corpuser:foo"
     with patch("requests.Session.get") as mock_get:
         mock_response = Mock()
diff --git a/metadata-ingestion/tests/unit/test_kafka_emitter.py b/metadata-ingestion/tests/unit/sdk/test_kafka_emitter.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_kafka_emitter.py
rename to metadata-ingestion/tests/unit/sdk/test_kafka_emitter.py
diff --git a/metadata-ingestion/tests/unit/test_mce_builder.py b/metadata-ingestion/tests/unit/sdk/test_mce_builder.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_mce_builder.py
rename to metadata-ingestion/tests/unit/sdk/test_mce_builder.py
diff --git a/metadata-ingestion/tests/unit/test_mcp_builder.py b/metadata-ingestion/tests/unit/sdk/test_mcp_builder.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_mcp_builder.py
rename to metadata-ingestion/tests/unit/sdk/test_mcp_builder.py
diff --git a/metadata-ingestion/tests/unit/test_mcp_wrapper.py b/metadata-ingestion/tests/unit/sdk/test_mcp_wrapper.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_mcp_wrapper.py
rename to metadata-ingestion/tests/unit/sdk/test_mcp_wrapper.py
diff --git a/metadata-ingestion/tests/unit/test_rest_emitter.py b/metadata-ingestion/tests/unit/sdk/test_rest_emitter.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_rest_emitter.py
rename to metadata-ingestion/tests/unit/sdk/test_rest_emitter.py
diff --git a/metadata-ingestion/tests/unit/test_codegen.py b/metadata-ingestion/tests/unit/serde/test_codegen.py
similarity index 95%
rename from metadata-ingestion/tests/unit/test_codegen.py
rename to metadata-ingestion/tests/unit/serde/test_codegen.py
index 37ac35586950e..98d62d5643ff2 100644
--- a/metadata-ingestion/tests/unit/test_codegen.py
+++ b/metadata-ingestion/tests/unit/serde/test_codegen.py
@@ -18,6 +18,7 @@
     UpstreamClass,
     _Aspect,
 )
+from datahub.utilities.urns._urn_base import URN_TYPES
 
 _UPDATE_ENTITY_REGISTRY = os.getenv("UPDATE_ENTITY_REGISTRY", "false").lower() == "true"
 ENTITY_REGISTRY_PATH = pathlib.Path(
@@ -165,3 +166,9 @@ def test_enum_options():
     # This is mainly a sanity check to ensure that it doesn't do anything too crazy.
     env_options = get_enum_options(FabricTypeClass)
     assert "PROD" in env_options
+
+
+def test_urn_types() -> None:
+    assert len(URN_TYPES) > 10
+    for checked_type in ["dataset", "dashboard", "dataFlow", "schemaField"]:
+        assert checked_type in URN_TYPES
diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py
index 53ffdf46a6d1e..a131ac9ce2a1b 100644
--- a/metadata-ingestion/tests/unit/serde/test_serde.py
+++ b/metadata-ingestion/tests/unit/serde/test_serde.py
@@ -19,7 +19,6 @@
 from datahub.metadata.schemas import getMetadataChangeEventSchema
 from tests.test_helpers import mce_helpers
 from tests.test_helpers.click_helpers import run_datahub_cmd
-from tests.test_helpers.type_helpers import PytestConfig
 
 FROZEN_TIME = "2021-07-22 18:54:06"
 
@@ -41,7 +40,7 @@
     ],
 )
 def test_serde_to_json(
-    pytestconfig: PytestConfig, tmp_path: pathlib.Path, json_filename: str
+    pytestconfig: pytest.Config, tmp_path: pathlib.Path, json_filename: str
 ) -> None:
     golden_file = pytestconfig.rootpath / json_filename
     output_file = tmp_path / "output.json"
@@ -73,15 +72,13 @@ def test_serde_to_json(
 )
 @freeze_time(FROZEN_TIME)
 def test_serde_to_avro(
-    pytestconfig: PytestConfig,
+    pytestconfig: pytest.Config,
     json_filename: str,
 ) -> None:
     # In this test, we want to read in from JSON -> MCE object.
     # Next we serialize from MCE to Avro and then deserialize back to MCE.
     # Finally, we want to compare the two MCE objects.
-    with patch(
-        "datahub.ingestion.api.common.PipelineContext", autospec=True
-    ) as mock_pipeline_context:
+    with patch("datahub.ingestion.api.common.PipelineContext") as mock_pipeline_context:
         json_path = pytestconfig.rootpath / json_filename
         source = GenericFileSource(
             ctx=mock_pipeline_context, config=FileSourceConfig(path=str(json_path))
@@ -128,14 +125,14 @@ def test_serde_to_avro(
     ],
 )
 @freeze_time(FROZEN_TIME)
-def test_check_metadata_schema(pytestconfig: PytestConfig, json_filename: str) -> None:
+def test_check_metadata_schema(pytestconfig: pytest.Config, json_filename: str) -> None:
     json_file_path = pytestconfig.rootpath / json_filename
 
     run_datahub_cmd(["check", "metadata-file", f"{json_file_path}"])
 
 
 def test_check_metadata_rewrite(
-    pytestconfig: PytestConfig, tmp_path: pathlib.Path
+    pytestconfig: pytest.Config, tmp_path: pathlib.Path
 ) -> None:
     json_input = (
         pytestconfig.rootpath / "tests/unit/serde/test_canonicalization_input.json"
@@ -163,7 +160,7 @@ def test_check_metadata_rewrite(
     ],
 )
 def test_check_mce_schema_failure(
-    pytestconfig: PytestConfig, json_filename: str
+    pytestconfig: pytest.Config, json_filename: str
 ) -> None:
     json_file_path = pytestconfig.rootpath / json_filename
 
diff --git a/metadata-ingestion/tests/unit/test_snowflake_shares.py b/metadata-ingestion/tests/unit/snowflake/test_snowflake_shares.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_snowflake_shares.py
rename to metadata-ingestion/tests/unit/snowflake/test_snowflake_shares.py
diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py
similarity index 88%
rename from metadata-ingestion/tests/unit/test_snowflake_source.py
rename to metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py
index 72b59a3a4e493..2ff85a08f052f 100644
--- a/metadata-ingestion/tests/unit/test_snowflake_source.py
+++ b/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py
@@ -4,8 +4,8 @@
 import pytest
 from pydantic import ValidationError
 
+import datahub.ingestion.source.snowflake.snowflake_utils
 from datahub.configuration.common import AllowDenyPattern
-from datahub.configuration.oauth import OAuthConfiguration
 from datahub.configuration.pattern_utils import UUID_REGEX
 from datahub.ingestion.api.source import SourceCapability
 from datahub.ingestion.source.snowflake.constants import (
@@ -13,10 +13,12 @@
     CLIENT_SESSION_KEEP_ALIVE,
     SnowflakeCloudProvider,
 )
+from datahub.ingestion.source.snowflake.oauth_config import OAuthConfiguration
 from datahub.ingestion.source.snowflake.snowflake_config import (
     DEFAULT_TEMP_TABLES_PATTERNS,
     SnowflakeV2Config,
 )
+from datahub.ingestion.source.snowflake.snowflake_lineage_v2 import UpstreamLineageEdge
 from datahub.ingestion.source.snowflake.snowflake_query import (
     SnowflakeQuery,
     create_deny_regex_sql_filter,
@@ -26,6 +28,7 @@
 )
 from datahub.ingestion.source.snowflake.snowflake_utils import SnowsightUrlBuilder
 from datahub.ingestion.source.snowflake.snowflake_v2 import SnowflakeV2Source
+from datahub.testing.doctest import assert_doctest
 from tests.test_helpers import test_connection_helpers
 
 default_oauth_dict: Dict[str, Any] = {
@@ -130,6 +133,60 @@ def test_snowflake_oauth_happy_paths():
     )
 
 
+def test_snowflake_oauth_token_happy_path():
+    assert SnowflakeV2Config.parse_obj(
+        {
+            "account_id": "test",
+            "authentication_type": "OAUTH_AUTHENTICATOR_TOKEN",
+            "token": "valid-token",
+            "username": "test-user",
+            "oauth_config": None,
+        }
+    )
+
+
+def test_snowflake_oauth_token_without_token():
+    with pytest.raises(
+        ValidationError, match="Token required for OAUTH_AUTHENTICATOR_TOKEN."
+    ):
+        SnowflakeV2Config.parse_obj(
+            {
+                "account_id": "test",
+                "authentication_type": "OAUTH_AUTHENTICATOR_TOKEN",
+                "username": "test-user",
+            }
+        )
+
+
+def test_snowflake_oauth_token_with_wrong_auth_type():
+    with pytest.raises(
+        ValueError,
+        match="Token can only be provided when using OAUTH_AUTHENTICATOR_TOKEN.",
+    ):
+        SnowflakeV2Config.parse_obj(
+            {
+                "account_id": "test",
+                "authentication_type": "OAUTH_AUTHENTICATOR",
+                "token": "some-token",
+                "username": "test-user",
+            }
+        )
+
+
+def test_snowflake_oauth_token_with_empty_token():
+    with pytest.raises(
+        ValidationError, match="Token required for OAUTH_AUTHENTICATOR_TOKEN."
+    ):
+        SnowflakeV2Config.parse_obj(
+            {
+                "account_id": "test",
+                "authentication_type": "OAUTH_AUTHENTICATOR_TOKEN",
+                "token": "",
+                "username": "test-user",
+            }
+        )
+
+
 default_config_dict: Dict[str, Any] = {
     "username": "user",
     "password": "password",
@@ -604,3 +661,30 @@ def test_create_snowsight_base_url_ap_northeast_1():
     ).snowsight_base_url
 
     assert result == "https://app.snowflake.com/ap-northeast-1.aws/account_locator/"
+
+
+def test_snowflake_utils() -> None:
+    assert_doctest(datahub.ingestion.source.snowflake.snowflake_utils)
+
+
+def test_snowflake_query_result_parsing():
+    db_row = {
+        "DOWNSTREAM_TABLE_NAME": "db.schema.downstream_table",
+        "DOWNSTREAM_TABLE_DOMAIN": "Table",
+        "UPSTREAM_TABLES": [
+            {
+                "query_id": "01b92f61-0611-c826-000d-0103cf9b5db7",
+                "upstream_object_domain": "Table",
+                "upstream_object_name": "db.schema.upstream_table",
+            }
+        ],
+        "UPSTREAM_COLUMNS": [{}],
+        "QUERIES": [
+            {
+                "query_id": "01b92f61-0611-c826-000d-0103cf9b5db7",
+                "query_text": "Query test",
+                "start_time": "2022-12-01 19:56:34",
+            }
+        ],
+    }
+    assert UpstreamLineageEdge.parse_obj(db_row)
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json
index 94c8947dba9ff..0d8822736c95e 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json
@@ -85,26 +85,6 @@
         }
     }
 },
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1707182625000,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "operationType": "INSERT",
-            "customProperties": {
-                "query_urn": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f"
-            },
-            "lastUpdatedTimestamp": 20000
-        }
-    }
-},
 {
     "entityType": "query",
     "entityUrn": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f",
@@ -142,5 +122,25 @@
             "platform": "urn:li:dataPlatform:redshift"
         }
     }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "operation",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1707182625000,
+            "partitionSpec": {
+                "partition": "FULL_TABLE_SNAPSHOT",
+                "type": "FULL_TABLE"
+            },
+            "operationType": "INSERT",
+            "customProperties": {
+                "query_urn": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f"
+            },
+            "lastUpdatedTimestamp": 20000
+        }
+    }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json
index d3ec384316818..290ee7091df49 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json
@@ -164,10 +164,10 @@
                     "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)"
                 },
                 {
-                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),c)"
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)"
                 },
                 {
-                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)"
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),c)"
                 },
                 {
                     "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)"
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json
index a9b5a3a7cbefa..83d81037a2136 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json
@@ -18,7 +18,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream1,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45"
+                    "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a"
                 },
                 {
                     "auditStamp": {
@@ -31,7 +31,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream2,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45"
+                    "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a"
                 }
             ],
             "fineGrainedLineages": [
@@ -45,7 +45,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),a)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45"
+                    "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a"
                 },
                 {
                     "upstreamType": "FIELD_SET",
@@ -57,7 +57,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),b)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45"
+                    "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a"
                 },
                 {
                     "upstreamType": "FIELD_SET",
@@ -69,7 +69,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),c)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45"
+                    "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a"
                 }
             ]
         }
@@ -77,7 +77,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45",
+    "entityUrn": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a",
     "changeType": "UPSERT",
     "aspectName": "queryProperties",
     "aspect": {
@@ -100,7 +100,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45",
+    "entityUrn": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a",
     "changeType": "UPSERT",
     "aspectName": "querySubjects",
     "aspect": {
@@ -130,7 +130,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45",
+    "entityUrn": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json
index 48aecb9015180..ed8009477bf4d 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json
@@ -18,7 +18,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.customer,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17"
+                    "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8"
                 },
                 {
                     "auditStamp": {
@@ -31,7 +31,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.online_returns,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17"
+                    "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8"
                 },
                 {
                     "auditStamp": {
@@ -44,7 +44,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.online_survey,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17"
+                    "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8"
                 },
                 {
                     "auditStamp": {
@@ -57,7 +57,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.in_person_returns,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75"
+                    "query": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45"
                 }
             ],
             "fineGrainedLineages": [
@@ -71,7 +71,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_id)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75"
+                    "query": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45"
                 },
                 {
                     "upstreamType": "FIELD_SET",
@@ -83,7 +83,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_id)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17"
+                    "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8"
                 },
                 {
                     "upstreamType": "FIELD_SET",
@@ -95,7 +95,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_email)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17"
+                    "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8"
                 },
                 {
                     "upstreamType": "FIELD_SET",
@@ -107,7 +107,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_date)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75"
+                    "query": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45"
                 },
                 {
                     "upstreamType": "FIELD_SET",
@@ -119,7 +119,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_date)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17"
+                    "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8"
                 },
                 {
                     "upstreamType": "FIELD_SET",
@@ -131,7 +131,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_reason)"
                     ],
                     "confidenceScore": 0.2,
-                    "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17"
+                    "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8"
                 }
             ]
         }
@@ -139,7 +139,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17",
+    "entityUrn": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8",
     "changeType": "UPSERT",
     "aspectName": "queryProperties",
     "aspect": {
@@ -162,7 +162,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17",
+    "entityUrn": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8",
     "changeType": "UPSERT",
     "aspectName": "querySubjects",
     "aspect": {
@@ -198,7 +198,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17",
+    "entityUrn": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
@@ -209,7 +209,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75",
+    "entityUrn": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45",
     "changeType": "UPSERT",
     "aspectName": "queryProperties",
     "aspect": {
@@ -232,7 +232,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75",
+    "entityUrn": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45",
     "changeType": "UPSERT",
     "aspectName": "querySubjects",
     "aspect": {
@@ -262,7 +262,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75",
+    "entityUrn": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_lineage_via_temp_table_disordered_add.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_lineage_via_temp_table_disordered_add.json
index 51a732e7a2494..915718525dbc7 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_lineage_via_temp_table_disordered_add.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_lineage_via_temp_table_disordered_add.json
@@ -18,7 +18,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_39f4adf89c8ad4d6d307b628c82d8260e1c5cd7eb6fb3a8cbb437421f970c16f"
+                    "query": "urn:li:query:composite_cb80bd1f17cc5d3d956e03a82d4266ec0dc698ca4333547281fb93bf67d832ec"
                 }
             ]
         }
@@ -26,7 +26,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_39f4adf89c8ad4d6d307b628c82d8260e1c5cd7eb6fb3a8cbb437421f970c16f",
+    "entityUrn": "urn:li:query:composite_cb80bd1f17cc5d3d956e03a82d4266ec0dc698ca4333547281fb93bf67d832ec",
     "changeType": "UPSERT",
     "aspectName": "queryProperties",
     "aspect": {
@@ -49,7 +49,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_39f4adf89c8ad4d6d307b628c82d8260e1c5cd7eb6fb3a8cbb437421f970c16f",
+    "entityUrn": "urn:li:query:composite_cb80bd1f17cc5d3d956e03a82d4266ec0dc698ca4333547281fb93bf67d832ec",
     "changeType": "UPSERT",
     "aspectName": "querySubjects",
     "aspect": {
@@ -67,7 +67,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_39f4adf89c8ad4d6d307b628c82d8260e1c5cd7eb6fb3a8cbb437421f970c16f",
+    "entityUrn": "urn:li:query:composite_cb80bd1f17cc5d3d956e03a82d4266ec0dc698ca4333547281fb93bf67d832ec",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json
index 9a4d405e50a7a..fd8475090f009 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json
@@ -119,6 +119,245 @@
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:88d742bcc0216d6ccb50c7430d1d97494d5dfcfa90160ffa123108844ad261e4"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)"
+                    ],
+                    "confidenceScore": 1.0,
+                    "query": "urn:li:query:88d742bcc0216d6ccb50c7430d1d97494d5dfcfa90160ffa123108844ad261e4"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)"
+                    ],
+                    "confidenceScore": 1.0,
+                    "query": "urn:li:query:88d742bcc0216d6ccb50c7430d1d97494d5dfcfa90160ffa123108844ad261e4"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),c)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)"
+                    ],
+                    "confidenceScore": 1.0,
+                    "query": "urn:li:query:88d742bcc0216d6ccb50c7430d1d97494d5dfcfa90160ffa123108844ad261e4"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:88d742bcc0216d6ccb50c7430d1d97494d5dfcfa90160ffa123108844ad261e4",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "ALTER TABLE dev.public.foo_staging RENAME TO foo /* Datahub generated query text-- */",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:88d742bcc0216d6ccb50c7430d1d97494d5dfcfa90160ffa123108844ad261e4",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:88d742bcc0216d6ccb50c7430d1d97494d5dfcfa90160ffa123108844ad261e4",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:redshift"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE foo_downstream AS\nSELECT\n  a,\n  b\nFROM foo_staging",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:redshift"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
             "upstreams": [
@@ -144,7 +383,7 @@
                     ],
                     "downstreamType": "FIELD",
                     "downstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)"
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)"
                     ],
                     "confidenceScore": 0.2,
                     "query": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc"
@@ -156,7 +395,7 @@
                     ],
                     "downstreamType": "FIELD",
                     "downstreams": [
-                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)"
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)"
                     ],
                     "confidenceScore": 0.2,
                     "query": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc"
@@ -206,13 +445,13 @@
                     "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),b)"
                 },
                 {
-                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)"
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)"
                 },
                 {
-                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)"
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)"
                 },
                 {
-                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)"
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)"
                 }
             ]
         }
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json
new file mode 100644
index 0000000000000..a4ac349c3c455
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json
@@ -0,0 +1,338 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE bar AS\nSELECT\n  a,\n  b\nFROM baz",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),b)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:redshift"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE foo_staging AS\nSELECT\n  a,\n  b\nFROM foo_dep;\n\nALTER TABLE dev.public.foo_staging RENAME TO foo",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:redshift"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE foo_staging AS\nSELECT\n  a,\n  b\nFROM foo_dep;\n\nCREATE TABLE foo_downstream AS\nSELECT\n  a,\n  b\nFROM foo_staging",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:redshift"
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json
new file mode 100644
index 0000000000000..d9d46a4b14a14
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json
@@ -0,0 +1,537 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),a)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),b)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE bar AS\nSELECT\n  a,\n  b\nFROM baz",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),b)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),b)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:b256c8cc8f386b209ef8da55485d46c3fbd471b942f804d370e24350b3087405"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)"
+                    ],
+                    "confidenceScore": 1.0,
+                    "query": "urn:li:query:b256c8cc8f386b209ef8da55485d46c3fbd471b942f804d370e24350b3087405"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD),b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),b)"
+                    ],
+                    "confidenceScore": 1.0,
+                    "query": "urn:li:query:b256c8cc8f386b209ef8da55485d46c3fbd471b942f804d370e24350b3087405"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD),c)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),c)"
+                    ],
+                    "confidenceScore": 1.0,
+                    "query": "urn:li:query:b256c8cc8f386b209ef8da55485d46c3fbd471b942f804d370e24350b3087405"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:b256c8cc8f386b209ef8da55485d46c3fbd471b942f804d370e24350b3087405",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "ALTER TABLE dev.public.person_info_swap SWAP WITH dev.public.person_info /* Datahub generated query text-- */",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:b256c8cc8f386b209ef8da55485d46c3fbd471b942f804d370e24350b3087405",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),b)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),c)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:b256c8cc8f386b209ef8da55485d46c3fbd471b942f804d370e24350b3087405",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE person_info_backup AS\nSELECT\n  *\nFROM person_info_swap",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:4b1fad909083e1ed5c47c146bd01247ed4d6295d175c34f9065b8fc6000fc7ae"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:4b1fad909083e1ed5c47c146bd01247ed4d6295d175c34f9065b8fc6000fc7ae",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE person_info_incremental AS\nSELECT\n  *\nFROM person_info_dep",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:4b1fad909083e1ed5c47c146bd01247ed4d6295d175c34f9065b8fc6000fc7ae",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:4b1fad909083e1ed5c47c146bd01247ed4d6295d175c34f9065b8fc6000fc7ae",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:3886d427c84692923797048da6d3991693e89ce44e10d1917c12e8b6fd493904"
+                },
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:3886d427c84692923797048da6d3991693e89ce44e10d1917c12e8b6fd493904",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "ALTER TABLE dev.public.person_info SWAP WITH dev.public.person_info_swap /* Datahub generated query text-- */",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:3886d427c84692923797048da6d3991693e89ce44e10d1917c12e8b6fd493904",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:3886d427c84692923797048da6d3991693e89ce44e10d1917c12e8b6fd493904",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "INSERT INTO person_info_swap\nSELECT\n  *\nFROM person_info_incremental",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json
new file mode 100644
index 0000000000000..b4eaf76a14933
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json
@@ -0,0 +1,342 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),a)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),b)"
+                    ],
+                    "confidenceScore": 0.2,
+                    "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE bar AS\nSELECT\n  a,\n  b\nFROM baz",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),b)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),b)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 40000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:composite_9e36ef19163461d35b618fd1eea2a3f6a5d10a23a979a6d5ef688b31f277abb3"
+                },
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 40000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:composite_9e36ef19163461d35b618fd1eea2a3f6a5d10a23a979a6d5ef688b31f277abb3"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)"
+                    ],
+                    "confidenceScore": 1.0,
+                    "query": "urn:li:query:composite_9e36ef19163461d35b618fd1eea2a3f6a5d10a23a979a6d5ef688b31f277abb3"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_9e36ef19163461d35b618fd1eea2a3f6a5d10a23a979a6d5ef688b31f277abb3",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE person_info_swap CLONE person_info;\n\nCREATE TABLE person_info_incremental AS\nSELECT\n  *\nFROM person_info_dep;\n\nINSERT INTO person_info_swap\nSELECT\n  *\nFROM person_info_incremental;\n\nALTER TABLE dev.public.person_info_swap SWAP WITH dev.public.person_info /* Datahub generated query text-- */",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 40000,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 40000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_9e36ef19163461d35b618fd1eea2a3f6a5d10a23a979a6d5ef688b31f277abb3",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),b)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),c)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_9e36ef19163461d35b618fd1eea2a3f6a5d10a23a979a6d5ef688b31f277abb3",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 50000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80"
+                },
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 50000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD),a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD),a)"
+                    ],
+                    "confidenceScore": 1.0,
+                    "query": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE person_info_swap CLONE person_info;\n\nCREATE TABLE person_info_incremental AS\nSELECT\n  *\nFROM person_info_dep;\n\nINSERT INTO person_info_swap\nSELECT\n  *\nFROM person_info_incremental;\n\nCREATE TABLE person_info_backup AS\nSELECT\n  *\nFROM person_info_swap",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 50000,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 50000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)"
+                },
+                {
+                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD),a)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp_with_preparsed.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp_with_preparsed.json
new file mode 100644
index 0000000000000..24ed6a3b54d8c
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp_with_preparsed.json
@@ -0,0 +1,298 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "ALTER TABLE dev.public.person_info_swap SWAP WITH dev.public.person_info",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "CREATE TABLE person_info_backup AS\nSELECT\n  *\nFROM person_info_swap",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68"
+                },
+                {
+                    "auditStamp": {
+                        "time": 1707182625000,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:_ingestion"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)",
+                    "type": "TRANSFORMED",
+                    "query": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "ALTER TABLE dev.public.person_info SWAP WITH dev.public.person_info_swap",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f",
+    "changeType": "UPSERT",
+    "aspectName": "queryProperties",
+    "aspect": {
+        "json": {
+            "statement": {
+                "value": "INSERT INTO person_info_swap\nSELECT\n  *\nFROM person_info_incremental",
+                "language": "SQL"
+            },
+            "source": "SYSTEM",
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:_ingestion"
+            },
+            "lastModified": {
+                "time": 1707182625000,
+                "actor": "urn:li:corpuser:_ingestion"
+            }
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f",
+    "changeType": "UPSERT",
+    "aspectName": "querySubjects",
+    "aspect": {
+        "json": {
+            "subjects": [
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)"
+                },
+                {
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)"
+                }
+            ]
+        }
+    }
+},
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake"
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json
index 743e2738fc70c..bcd31b0aa0249 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json
@@ -133,7 +133,7 @@
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)",
                     "type": "TRANSFORMED",
-                    "query": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec"
+                    "query": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817"
                 }
             ],
             "fineGrainedLineages": [
@@ -147,7 +147,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session2,PROD),a)"
                     ],
                     "confidenceScore": 0.35,
-                    "query": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec"
+                    "query": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817"
                 },
                 {
                     "upstreamType": "FIELD_SET",
@@ -160,7 +160,7 @@
                         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session2,PROD),c)"
                     ],
                     "confidenceScore": 0.35,
-                    "query": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec"
+                    "query": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817"
                 }
             ]
         }
@@ -168,7 +168,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec",
+    "entityUrn": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817",
     "changeType": "UPSERT",
     "aspectName": "queryProperties",
     "aspect": {
@@ -191,7 +191,7 @@
 },
 {
     "entityType": "query",
-    "entityUrn": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec",
+    "entityUrn": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817",
     "changeType": "UPSERT",
     "aspectName": "querySubjects",
     "aspect": {
@@ -213,6 +213,17 @@
         }
     }
 },
+{
+    "entityType": "query",
+    "entityUrn": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:redshift"
+        }
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session3,PROD)",
@@ -238,17 +249,6 @@
         }
     }
 },
-{
-    "entityType": "query",
-    "entityUrn": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec",
-    "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
-    "aspect": {
-        "json": {
-            "platform": "urn:li:dataPlatform:redshift"
-        }
-    }
-},
 {
     "entityType": "query",
     "entityUrn": "urn:li:query:3e85e6f353c7fa33d6514cb090482852064d23df6491c9a8ae28be0d990a3c71",
@@ -281,10 +281,10 @@
         "json": {
             "subjects": [
                 {
-                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session3,PROD)"
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)"
                 },
                 {
-                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)"
+                    "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session3,PROD)"
                 }
             ]
         }
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_information_schema_query.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_information_schema_query.json
index f5f573f3d5113..9621b7d1c265b 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_information_schema_query.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_information_schema_query.json
@@ -1,7 +1,7 @@
 {
     "query_type": "SELECT",
     "query_type_props": {},
-    "query_fingerprint": "c721ce16410601b36e5f32bd9c5c28488500a93e617363739faebfe71496f163",
+    "query_fingerprint": "a204522c98a01568d8575a98a715de98985aeef0e822feb8450153f71891d6c6",
     "in_tables": [
         "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-staging-2.smoke_test_db_4.INFORMATION_SCHEMA.COLUMNS,PROD)",
         "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-staging-2.smoke_test_db_4.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS,PROD)"
@@ -178,6 +178,6 @@
     ],
     "debug_info": {
         "confidence": 0.2,
-        "generalized_statement": "SELECT c.table_catalog AS table_catalog, c.table_schema AS table_schema, c.table_name AS table_name, c.column_name AS column_name, c.ordinal_position AS ordinal_position, cfp.field_path AS field_path, c.is_nullable AS is_nullable, CASE WHEN CONTAINS_SUBSTR(cfp.field_path, ?) THEN NULL ELSE c.data_type END AS data_type, description AS comment, c.is_hidden AS is_hidden, c.is_partitioning_column AS is_partitioning_column, c.clustering_ordinal_position AS clustering_ordinal_position FROM `acryl-staging-2`.`smoke_test_db_4`.INFORMATION_SCHEMA.COLUMNS AS c JOIN `acryl-staging-2`.`smoke_test_db_4`.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS AS cfp ON cfp.table_name = c.table_name AND cfp.column_name = c.column_name ORDER BY table_catalog, table_schema, table_name, ordinal_position ASC, data_type DESC"
+        "generalized_statement": "SELECT c.table_catalog AS table_catalog, c.table_schema AS table_schema, c.table_name AS table_name, c.column_name AS column_name, c.ordinal_position AS ordinal_position, cfp.field_path AS field_path, c.is_nullable AS is_nullable, CASE WHEN CONTAINS_SUBSTR(cfp.field_path, ?) THEN NULL ELSE c.data_type END AS data_type, description AS comment, c.is_hidden AS is_hidden, c.is_partitioning_column AS is_partitioning_column, c.clustering_ordinal_position AS clustering_ordinal_position FROM `acryl-staging-2`.`smoke_test_db_4`.`INFORMATION_SCHEMA.COLUMNS` AS c JOIN `acryl-staging-2`.`smoke_test_db_4`.`INFORMATION_SCHEMA.COLUMN_FIELD_PATHS` AS cfp ON cfp.table_name = c.table_name AND cfp.column_name = c.column_name ORDER BY table_catalog, table_schema, table_name, ordinal_position ASC, data_type DESC"
     }
 }
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_subquery_column_inference.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_subquery_column_inference.json
new file mode 100644
index 0000000000000..32b3830bf1412
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_subquery_column_inference.json
@@ -0,0 +1,57 @@
+{
+    "query_type": "SELECT",
+    "query_type_props": {},
+    "query_fingerprint": "4094ebd230c1d47c7e6879b05ab927e550923b1986eb58c5f3814396cf401d18",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:bigquery,invent_dw.UserDetail,PROD)"
+    ],
+    "out_tables": [],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": null,
+                "column": "user_id",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,invent_dw.UserDetail,PROD)",
+                    "column": "user_id"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "source",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,invent_dw.UserDetail,PROD)",
+                    "column": "source"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "user_source",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,invent_dw.UserDetail,PROD)",
+                    "column": "user_source"
+                }
+            ]
+        }
+    ],
+    "debug_info": {
+        "confidence": 0.2,
+        "generalized_statement": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY __partition_day DESC) AS rank_ FROM invent_dw.UserDetail) AS source_user WHERE rank_ = ?"
+    }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json
new file mode 100644
index 0000000000000..072260f5cd165
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json
@@ -0,0 +1,82 @@
+{
+    "query_type": "SELECT",
+    "query_type_props": {},
+    "query_fingerprint": "6a779a57ffb2598c301606d3a7d82142a7af8b102efa55a2c9a4e960fd55ac07",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)"
+    ],
+    "out_tables": [],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": null,
+                "column": "age",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INTEGER"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)",
+                    "column": "Age"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "name",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR(16777216)"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)",
+                    "column": "Name"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "uppercased_col",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR(16777216)"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)",
+                    "column": "Uppercased_Col"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "count",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INTEGER"
+            },
+            "upstreams": []
+        }
+    ],
+    "debug_info": {
+        "confidence": 0.9,
+        "generalized_statement": "SELECT Age, name, UPPERCASED_COL, COUNT(*) AS Count FROM Foo.Persons GROUP BY Age"
+    }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json
new file mode 100644
index 0000000000000..ff8609922a9bd
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json
@@ -0,0 +1,48 @@
+{
+    "query_type": "SELECT",
+    "query_type_props": {},
+    "query_fingerprint": "d18c43ed4d7f0e303b88c40ac078c5505be473edf0136e42a77dccc090d1eb8b",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)"
+    ],
+    "out_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)"
+    ],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)",
+                "column": "AGE",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INTEGER"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)",
+                    "column": "Age"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)",
+                "column": "Count",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INTEGER"
+            },
+            "upstreams": []
+        }
+    ],
+    "debug_info": {
+        "confidence": 0.9,
+        "generalized_statement": "SELECT age AS AGE, COUNT(*) AS Count INTO Foo.age_dist FROM Foo.Persons GROUP BY Age"
+    }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_attach_database.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_attach_database.json
new file mode 100644
index 0000000000000..bcf31f6be803a
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_attach_database.json
@@ -0,0 +1,12 @@
+{
+    "query_type": "UNKNOWN",
+    "query_type_props": {},
+    "query_fingerprint": null,
+    "in_tables": [],
+    "out_tables": [],
+    "column_lineage": null,
+    "debug_info": {
+        "confidence": 0.0,
+        "generalized_statement": null
+    }
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py b/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py
index 5a33034f274dc..67222531d3bc1 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py
@@ -1,7 +1,12 @@
-from datahub.sql_parsing.schema_resolver import SchemaResolver, _TableName
+from datahub.sql_parsing.schema_resolver import (
+    SchemaInfo,
+    SchemaResolver,
+    _TableName,
+    match_columns_to_schema,
+)
 
 
-def test_basic_schema_resolver():
+def create_default_schema_resolver(urn: str) -> SchemaResolver:
     schema_resolver = SchemaResolver(
         platform="redshift",
         env="PROD",
@@ -9,18 +14,51 @@ def test_basic_schema_resolver():
     )
 
     schema_resolver.add_raw_schema_info(
-        urn="urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.public.test_table,PROD)",
+        urn=urn,
         schema_info={"name": "STRING"},
     )
 
+    return schema_resolver
+
+
+def test_basic_schema_resolver():
+    input_urn = (
+        "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.public.test_table,PROD)"
+    )
+
+    schema_resolver = create_default_schema_resolver(urn=input_urn)
+
     urn, schema = schema_resolver.resolve_table(
         _TableName(database="my_db", db_schema="public", table="test_table")
     )
-    assert (
-        urn
-        == "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.public.test_table,PROD)"
+
+    assert urn == input_urn
+
+    assert schema
+
+    assert schema["name"]
+
+    assert schema_resolver.schema_count() == 1
+
+
+def test_resolve_urn():
+    input_urn: str = (
+        "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.public.test_table,PROD)"
     )
+
+    schema_resolver = create_default_schema_resolver(urn=input_urn)
+
+    schema_resolver.add_raw_schema_info(
+        urn=input_urn,
+        schema_info={"name": "STRING"},
+    )
+
+    urn, schema = schema_resolver.resolve_urn(urn=input_urn)
+
+    assert urn == input_urn
+
     assert schema
+
     assert schema["name"]
 
     assert schema_resolver.schema_count() == 1
@@ -41,6 +79,11 @@ def test_get_urn_for_table_lowercase():
         == "urn:li:dataset:(urn:li:dataPlatform:mssql,uppercased-instance.database.dataset.table,PROD)"
     )
 
+    assert (
+        schema_resolver.get_urn_for_table(table=table, lower=True, mixed=True)
+        == "urn:li:dataset:(urn:li:dataPlatform:mssql,Uppercased-Instance.database.dataset.table,PROD)"
+    )
+
 
 def test_get_urn_for_table_not_lower_should_keep_capital_letters():
     schema_resolver = SchemaResolver(
@@ -57,3 +100,13 @@ def test_get_urn_for_table_not_lower_should_keep_capital_letters():
         == "urn:li:dataset:(urn:li:dataPlatform:mssql,Uppercased-Instance.Database.DataSet.Table,PROD)"
     )
     assert schema_resolver.schema_count() == 0
+
+
+def test_match_columns_to_schema():
+    schema_info: SchemaInfo = {"id": "string", "Name": "string", "Address": "string"}
+
+    output_columns = match_columns_to_schema(
+        schema_info, input_columns=["Id", "name", "address", "weight"]
+    )
+
+    assert output_columns == ["id", "Name", "Address", "weight"]
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_split_statements.py b/metadata-ingestion/tests/unit/sql_parsing/test_split_statements.py
new file mode 100644
index 0000000000000..06e0e84ede554
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_split_statements.py
@@ -0,0 +1,51 @@
+from datahub.sql_parsing.split_statements import split_statements
+
+
+def test_split_statements_complex() -> None:
+    test_sql = """
+        CREATE TABLE Users (Id INT);
+        -- Comment here
+        INSERT INTO Users VALUES (1);
+        BEGIN
+            UPDATE Users SET Id = 2;
+            /* Multi-line
+               comment */
+            DELETE FROM /* inline DELETE comment */ Users;
+        END
+        GO
+        SELECT * FROM Users
+    """
+
+    statements = [statement.strip() for statement in split_statements(test_sql)]
+    assert statements == [
+        "CREATE TABLE Users (Id INT)",
+        "-- Comment here",
+        "INSERT INTO Users VALUES (1)",
+        "BEGIN",
+        "UPDATE Users SET Id = 2",
+        "/* Multi-line\n               comment */",
+        "DELETE FROM /* inline DELETE comment */ Users",
+        "END",
+        "GO",
+        "SELECT * FROM Users",
+    ]
+
+
+def test_split_statements_cte() -> None:
+    # SQL example from https://stackoverflow.com/a/11562724
+    test_sql = """\
+WITH T AS
+(   SELECT  InvoiceNumber,
+            DocTotal,
+            SUM(Sale + VAT) OVER(PARTITION BY InvoiceNumber) AS NewDocTotal
+    FROM    PEDI_InvoiceDetail
+)
+-- comment
+/* multi-line
+comment */
+UPDATE  T
+SET     DocTotal = NewDocTotal"""
+    statements = [statement.strip() for statement in split_statements(test_sql)]
+    assert statements == [
+        test_sql,
+    ]
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py
index 0d21936a74d07..2a771a9847abd 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py
@@ -1,5 +1,8 @@
+import functools
+import os
 import pathlib
 from datetime import datetime, timezone
+from unittest.mock import patch
 
 import pytest
 from freezegun import freeze_time
@@ -11,8 +14,11 @@
 from datahub.sql_parsing.sql_parsing_aggregator import (
     KnownQueryLineageInfo,
     ObservedQuery,
+    PreparsedQuery,
     QueryLogSetting,
     SqlParsingAggregator,
+    TableRename,
+    TableSwap,
 )
 from datahub.sql_parsing.sql_parsing_common import QueryType
 from datahub.sql_parsing.sqlglot_lineage import (
@@ -26,6 +32,10 @@
 RESOURCE_DIR = pathlib.Path(__file__).parent / "aggregator_goldens"
 FROZEN_TIME = "2024-02-06T01:23:45Z"
 
+check_goldens_stream = functools.partial(
+    mce_helpers.check_goldens_stream, ignore_order=False
+)
+
 
 def _ts(ts: int) -> datetime:
     return datetime.fromtimestamp(ts, tz=timezone.utc)
@@ -51,7 +61,7 @@ def test_basic_lineage(pytestconfig: pytest.Config, tmp_path: pathlib.Path) -> N
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_basic_lineage.json",
@@ -103,7 +113,7 @@ def test_overlapping_inserts(pytestconfig: pytest.Config) -> None:
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_overlapping_inserts.json",
@@ -162,7 +172,7 @@ def test_temp_table(pytestconfig: pytest.Config) -> None:
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_temp_table.json",
@@ -219,13 +229,12 @@ def test_multistep_temp_table(pytestconfig: pytest.Config) -> None:
     assert (
         len(
             report.queries_with_temp_upstreams[
-                "composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45"
+                "composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a"
             ]
         )
         == 4
     )
-
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_multistep_temp_table.json",
@@ -301,7 +310,7 @@ def test_overlapping_inserts_from_temp_tables(pytestconfig: pytest.Config) -> No
     assert len(report.queries_with_non_authoritative_session) == 1
 
     mcps = list(aggregator.gen_metadata())
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_overlapping_inserts_from_temp_tables.json",
@@ -350,7 +359,7 @@ def test_aggregate_operations(pytestconfig: pytest.Config) -> None:
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_aggregate_operations.json",
@@ -388,7 +397,7 @@ def test_view_lineage(pytestconfig: pytest.Config) -> None:
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_view_lineage.json",
@@ -419,7 +428,7 @@ def test_known_lineage_mapping(pytestconfig: pytest.Config) -> None:
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_known_lineage_mapping.json",
@@ -457,7 +466,7 @@ def test_column_lineage_deduplication(pytestconfig: pytest.Config) -> None:
     # not get any credit for a and b, as they are already covered by query 2,
     # which came later and hence has higher precedence.
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_column_lineage_deduplication.json",
@@ -502,7 +511,7 @@ def test_add_known_query_lineage(pytestconfig: pytest.Config) -> None:
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_add_known_query_lineage.json",
@@ -518,10 +527,17 @@ def test_table_rename(pytestconfig: pytest.Config) -> None:
         generate_operations=False,
     )
 
+    aggregator._schema_resolver.add_raw_schema_info(
+        DatasetUrn("redshift", "dev.public.foo").urn(),
+        {"a": "int", "b": "int", "c": "int"},
+    )
+
     # Register that foo_staging is renamed to foo.
     aggregator.add_table_rename(
-        original_urn=DatasetUrn("redshift", "dev.public.foo_staging").urn(),
-        new_urn=DatasetUrn("redshift", "dev.public.foo").urn(),
+        TableRename(
+            original_urn=DatasetUrn("redshift", "dev.public.foo_staging").urn(),
+            new_urn=DatasetUrn("redshift", "dev.public.foo").urn(),
+        )
     )
 
     # Add an unrelated query.
@@ -542,15 +558,341 @@ def test_table_rename(pytestconfig: pytest.Config) -> None:
         )
     )
 
+    # Add the query that created the downstream from foo_staging table.
+    aggregator.add_observed_query(
+        ObservedQuery(
+            query="create table foo_downstream as select a, b from foo_staging",
+            default_db="dev",
+            default_schema="public",
+        )
+    )
+
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_table_rename.json",
     )
 
 
+@freeze_time(FROZEN_TIME)
+def test_table_rename_with_temp(pytestconfig: pytest.Config) -> None:
+    aggregator = SqlParsingAggregator(
+        platform="redshift",
+        generate_lineage=True,
+        generate_usage_statistics=False,
+        generate_operations=False,
+        is_temp_table=lambda x: "staging" in x.lower(),
+    )
+
+    aggregator._schema_resolver.add_raw_schema_info(
+        DatasetUrn("redshift", "dev.public.foo").urn(),
+        {"a": "int", "b": "int", "c": "int"},
+    )
+
+    # Register that foo_staging is renamed to foo.
+    aggregator.add_table_rename(
+        TableRename(
+            original_urn=DatasetUrn("redshift", "dev.public.foo_staging").urn(),
+            new_urn=DatasetUrn("redshift", "dev.public.foo").urn(),
+            query="alter table dev.public.foo_staging rename to dev.public.foo",
+        )
+    )
+
+    # Add an unrelated query.
+    aggregator.add_observed_query(
+        ObservedQuery(
+            query="create table bar as select a, b from baz",
+            default_db="dev",
+            default_schema="public",
+        )
+    )
+
+    # Add the query that created the staging table.
+    aggregator.add_observed_query(
+        ObservedQuery(
+            query="create table foo_staging as select a, b from foo_dep",
+            default_db="dev",
+            default_schema="public",
+        )
+    )
+
+    # Add the query that created the downstream from foo_staging table.
+    aggregator.add_observed_query(
+        ObservedQuery(
+            query="create table foo_downstream as select a, b from foo_staging",
+            default_db="dev",
+            default_schema="public",
+        )
+    )
+
+    mcps = list(aggregator.gen_metadata())
+
+    check_goldens_stream(
+        pytestconfig,
+        outputs=mcps,
+        golden_path=RESOURCE_DIR / "test_table_rename_with_temp.json",
+    )
+
+
+@freeze_time(FROZEN_TIME)
+def test_table_swap(pytestconfig: pytest.Config) -> None:
+    aggregator = SqlParsingAggregator(
+        platform="snowflake",
+        generate_lineage=True,
+        generate_usage_statistics=False,
+        generate_operations=False,
+    )
+
+    aggregator._schema_resolver.add_raw_schema_info(
+        DatasetUrn("snowflake", "dev.public.person_info").urn(),
+        {"a": "int", "b": "int", "c": "int"},
+    )
+
+    # Add an unrelated query.
+    aggregator.add_observed_query(
+        ObservedQuery(
+            query="create table bar as select a, b from baz",
+            default_db="dev",
+            default_schema="public",
+        )
+    )
+
+    # Add the query that created the swap table initially.
+    aggregator.add_preparsed_query(
+        PreparsedQuery(
+            query_id=None,
+            query_text="CREATE TABLE person_info_swap CLONE person_info;",
+            upstreams=[DatasetUrn("snowflake", "dev.public.person_info").urn()],
+            downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(),
+        )
+    )
+
+    # Add the query that created the incremental table.
+    aggregator.add_preparsed_query(
+        PreparsedQuery(
+            query_id=None,
+            query_text="CREATE TABLE person_info_incremental AS SELECT * from person_info_dep;",
+            upstreams=[
+                DatasetUrn("snowflake", "dev.public.person_info_dep").urn(),
+            ],
+            downstream=DatasetUrn(
+                "snowflake", "dev.public.person_info_incremental"
+            ).urn(),
+        )
+    )
+
+    # Add the query that updated the swap table.
+    aggregator.add_preparsed_query(
+        PreparsedQuery(
+            query_id=None,
+            query_text="INSERT INTO person_info_swap SELECT * from person_info_incremental;",
+            upstreams=[
+                DatasetUrn("snowflake", "dev.public.person_info_incremental").urn(),
+            ],
+            downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(),
+        )
+    )
+
+    aggregator.add_table_swap(
+        TableSwap(
+            urn1=DatasetUrn("snowflake", "dev.public.person_info").urn(),
+            urn2=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(),
+        )
+    )
+
+    # Add the query that is created from swap table.
+    aggregator.add_preparsed_query(
+        PreparsedQuery(
+            query_id=None,
+            query_text="create table person_info_backup as select * from person_info_swap",
+            upstreams=[
+                DatasetUrn("snowflake", "dev.public.person_info_swap").urn(),
+            ],
+            downstream=DatasetUrn("snowflake", "dev.public.person_info_backup").urn(),
+        )
+    )
+
+    mcps = list(aggregator.gen_metadata())
+
+    check_goldens_stream(
+        pytestconfig,
+        outputs=mcps,
+        golden_path=RESOURCE_DIR / "test_table_swap.json",
+    )
+
+
+@freeze_time(FROZEN_TIME)
+def test_table_swap_with_temp(pytestconfig: pytest.Config) -> None:
+    aggregator = SqlParsingAggregator(
+        platform="snowflake",
+        generate_lineage=True,
+        generate_usage_statistics=False,
+        generate_operations=False,
+        is_temp_table=lambda x: "swap" in x.lower() or "incremental" in x.lower(),
+    )
+
+    aggregator._schema_resolver.add_raw_schema_info(
+        DatasetUrn("snowflake", "dev.public.person_info").urn(),
+        {"a": "int", "b": "int", "c": "int"},
+    )
+
+    # Add an unrelated query.
+    aggregator.add_observed_query(
+        ObservedQuery(
+            query="create table bar as select a, b from baz",
+            default_db="dev",
+            default_schema="public",
+        )
+    )
+
+    # Add the query that created the swap table initially.
+    aggregator.add_preparsed_query(
+        PreparsedQuery(
+            query_id=None,
+            query_text="CREATE TABLE person_info_swap CLONE person_info;",
+            upstreams=[DatasetUrn("snowflake", "dev.public.person_info").urn()],
+            downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(),
+            session_id="xxx",
+            timestamp=_ts(10),
+            column_lineage=[
+                ColumnLineageInfo(
+                    downstream=DownstreamColumnRef(
+                        table=DatasetUrn(
+                            "snowflake", "dev.public.person_info_swap"
+                        ).urn(),
+                        column="a",
+                    ),
+                    upstreams=[
+                        ColumnRef(
+                            table=DatasetUrn(
+                                "snowflake", "dev.public.person_info"
+                            ).urn(),
+                            column="a",
+                        )
+                    ],
+                )
+            ],
+        )
+    )
+
+    # Add the query that created the incremental table.
+    aggregator.add_preparsed_query(
+        PreparsedQuery(
+            query_id=None,
+            query_text="CREATE TABLE person_info_incremental AS SELECT * from person_info_dep;",
+            upstreams=[
+                DatasetUrn("snowflake", "dev.public.person_info_dep").urn(),
+            ],
+            downstream=DatasetUrn(
+                "snowflake", "dev.public.person_info_incremental"
+            ).urn(),
+            session_id="xxx",
+            timestamp=_ts(20),
+            column_lineage=[
+                ColumnLineageInfo(
+                    downstream=DownstreamColumnRef(
+                        table=DatasetUrn(
+                            "snowflake", "dev.public.person_info_incremental"
+                        ).urn(),
+                        column="a",
+                    ),
+                    upstreams=[
+                        ColumnRef(
+                            table=DatasetUrn(
+                                "snowflake", "dev.public.person_info_dep"
+                            ).urn(),
+                            column="a",
+                        )
+                    ],
+                )
+            ],
+        )
+    )
+
+    # Add the query that updated the swap table.
+    aggregator.add_preparsed_query(
+        PreparsedQuery(
+            query_id=None,
+            query_text="INSERT INTO person_info_swap SELECT * from person_info_incremental;",
+            upstreams=[
+                DatasetUrn("snowflake", "dev.public.person_info_incremental").urn(),
+            ],
+            downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(),
+            session_id="xxx",
+            timestamp=_ts(30),
+            column_lineage=[
+                ColumnLineageInfo(
+                    downstream=DownstreamColumnRef(
+                        table=DatasetUrn(
+                            "snowflake", "dev.public.person_info_swap"
+                        ).urn(),
+                        column="a",
+                    ),
+                    upstreams=[
+                        ColumnRef(
+                            table=DatasetUrn(
+                                "snowflake", "dev.public.person_info_incremental"
+                            ).urn(),
+                            column="a",
+                        )
+                    ],
+                )
+            ],
+        )
+    )
+
+    aggregator.add_table_swap(
+        TableSwap(
+            urn1=DatasetUrn("snowflake", "dev.public.person_info").urn(),
+            urn2=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(),
+            session_id="xxx",
+            timestamp=_ts(40),
+        )
+    )
+
+    # Add the query that is created from swap table.
+    aggregator.add_preparsed_query(
+        PreparsedQuery(
+            query_id=None,
+            query_text="create table person_info_backup as select * from person_info_swap",
+            upstreams=[
+                DatasetUrn("snowflake", "dev.public.person_info_swap").urn(),
+            ],
+            downstream=DatasetUrn("snowflake", "dev.public.person_info_backup").urn(),
+            session_id="xxx",
+            timestamp=_ts(50),
+            column_lineage=[
+                ColumnLineageInfo(
+                    downstream=DownstreamColumnRef(
+                        table=DatasetUrn(
+                            "snowflake", "dev.public.person_info_backup"
+                        ).urn(),
+                        column="a",
+                    ),
+                    upstreams=[
+                        ColumnRef(
+                            table=DatasetUrn(
+                                "snowflake", "dev.public.person_info_swap"
+                            ).urn(),
+                            column="a",
+                        )
+                    ],
+                )
+            ],
+        )
+    )
+
+    mcps = list(aggregator.gen_metadata())
+
+    check_goldens_stream(
+        pytestconfig,
+        outputs=mcps,
+        golden_path=RESOURCE_DIR / "test_table_swap_with_temp.json",
+    )
+
+
 @freeze_time(FROZEN_TIME)
 def test_create_table_query_mcps(pytestconfig: pytest.Config) -> None:
     aggregator = SqlParsingAggregator(
@@ -571,7 +913,7 @@ def test_create_table_query_mcps(pytestconfig: pytest.Config) -> None:
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_create_table_query_mcps.json",
@@ -606,7 +948,7 @@ def test_table_lineage_via_temp_table_disordered_add(
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR
@@ -616,7 +958,6 @@ def test_table_lineage_via_temp_table_disordered_add(
 
 @freeze_time(FROZEN_TIME)
 def test_basic_usage(pytestconfig: pytest.Config) -> None:
-
     frozen_timestamp = parse_user_datetime(FROZEN_TIME)
     aggregator = SqlParsingAggregator(
         platform="redshift",
@@ -656,8 +997,41 @@ def test_basic_usage(pytestconfig: pytest.Config) -> None:
 
     mcps = list(aggregator.gen_metadata())
 
-    mce_helpers.check_goldens_stream(
+    check_goldens_stream(
         pytestconfig,
         outputs=mcps,
         golden_path=RESOURCE_DIR / "test_basic_usage.json",
     )
+
+
+def test_table_swap_id() -> None:
+    assert (
+        TableSwap(
+            urn1=DatasetUrn("snowflake", "dev.public.foo").urn(),
+            urn2=DatasetUrn("snowflake", "dev.public.foo_staging").urn(),
+        ).id()
+        == TableSwap(
+            urn1=DatasetUrn("snowflake", "dev.public.foo_staging").urn(),
+            urn2=DatasetUrn("snowflake", "dev.public.foo").urn(),
+        ).id()
+    )
+
+
+def test_sql_aggreator_close_cleans_tmp(tmp_path):
+    frozen_timestamp = parse_user_datetime(FROZEN_TIME)
+    with patch("tempfile.tempdir", str(tmp_path)):
+        aggregator = SqlParsingAggregator(
+            platform="redshift",
+            generate_lineage=False,
+            generate_usage_statistics=True,
+            generate_operations=False,
+            usage_config=BaseUsageConfig(
+                start_time=get_time_bucket(frozen_timestamp, BucketDuration.DAY),
+                end_time=frozen_timestamp,
+            ),
+            generate_queries=True,
+            generate_query_usage_statistics=True,
+        )
+        assert len(os.listdir(tmp_path)) > 0
+        aggregator.close()
+        assert len(os.listdir(tmp_path)) == 0
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index fb1d2a0bc5011..2df6f6fa26cb5 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -1253,3 +1253,73 @@ def test_snowflake_drop_schema() -> None:
         dialect="snowflake",
         expected_file=RESOURCE_DIR / "test_snowflake_drop_schema.json",
     )
+
+
+def test_bigquery_subquery_column_inference() -> None:
+    assert_sql_result(
+        """\
+SELECT user_id, source, user_source
+FROM (
+    SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY __partition_day DESC) AS rank_
+    FROM invent_dw.UserDetail
+) source_user
+WHERE rank_ = 1
+""",
+        dialect="bigquery",
+        expected_file=RESOURCE_DIR / "test_bigquery_subquery_column_inference.json",
+    )
+
+
+def test_sqlite_attach_database() -> None:
+    assert_sql_result(
+        """\
+ATTACH DATABASE ':memory:' AS aux1
+""",
+        dialect="sqlite",
+        expected_file=RESOURCE_DIR / "test_sqlite_attach_database.json",
+        allow_table_error=True,
+    )
+
+
+def test_mssql_casing_resolver() -> None:
+    assert_sql_result(
+        """\
+SELECT Age, name, UPPERCASED_COL, COUNT(*) as Count
+FROM Foo.Persons
+GROUP BY Age
+""",
+        dialect="mssql",
+        default_db="NewData",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)": {
+                "Age": "INTEGER",
+                "Name": "VARCHAR(16777216)",
+                "Uppercased_Col": "VARCHAR(16777216)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_mssql_casing_resolver.json",
+    )
+
+
+def test_mssql_select_into() -> None:
+    assert_sql_result(
+        """\
+SELECT age as AGE, COUNT(*) as Count
+INTO Foo.age_dist
+FROM Foo.Persons
+GROUP BY Age
+""",
+        dialect="mssql",
+        default_db="NewData",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)": {
+                "Age": "INTEGER",
+                "Name": "VARCHAR(16777216)",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)": {
+                "AGE": "INTEGER",
+                "Count": "INTEGER",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_mssql_select_into.json",
+    )
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_patch.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_patch.py
new file mode 100644
index 0000000000000..dee6d9630c12e
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_patch.py
@@ -0,0 +1,48 @@
+from datahub.sql_parsing._sqlglot_patch import SQLGLOT_PATCHED
+
+import time
+
+import pytest
+import sqlglot
+import sqlglot.errors
+import sqlglot.lineage
+import sqlglot.optimizer
+
+from datahub.utilities.cooperative_timeout import (
+    CooperativeTimeoutError,
+    cooperative_timeout,
+)
+from datahub.utilities.perf_timer import PerfTimer
+
+assert SQLGLOT_PATCHED
+
+
+def test_cooperative_timeout_sql() -> None:
+    statement = sqlglot.parse_one("SELECT pg_sleep(3)", dialect="postgres")
+    with pytest.raises(
+        CooperativeTimeoutError
+    ), PerfTimer() as timer, cooperative_timeout(timeout=0.6):
+        while True:
+            # sql() implicitly calls copy(), which is where we check for the timeout.
+            assert statement.sql() is not None
+            time.sleep(0.0001)
+    assert 0.6 <= timer.elapsed_seconds() <= 1.0
+
+
+def test_scope_circular_dependency() -> None:
+    scope = sqlglot.optimizer.build_scope(
+        sqlglot.parse_one("WITH w AS (SELECT * FROM q) SELECT * FROM w")
+    )
+    assert scope is not None
+
+    cte_scope = scope.cte_scopes[0]
+    cte_scope.cte_scopes.append(cte_scope)
+
+    with pytest.raises(sqlglot.errors.OptimizeError, match="circular scope dependency"):
+        list(scope.traverse())
+
+
+def test_lineage_node_subfield() -> None:
+    expression = sqlglot.parse_one("SELECT 1 AS test")
+    node = sqlglot.lineage.Node("test", expression, expression, subfield="subfield")  # type: ignore
+    assert node.subfield == "subfield"  # type: ignore
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py
index 744d43373a0a1..dbe24ade6944f 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py
@@ -4,11 +4,9 @@
 import pytest
 import sqlglot
 
+from datahub.sql_parsing.query_types import get_query_type_of_sql
 from datahub.sql_parsing.sql_parsing_common import QueryType
-from datahub.sql_parsing.sqlglot_lineage import (
-    _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT,
-    get_query_type_of_sql,
-)
+from datahub.sql_parsing.sqlglot_lineage import _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT
 from datahub.sql_parsing.sqlglot_utils import (
     generalize_query,
     generalize_query_fast,
@@ -75,6 +73,12 @@ class QueryGeneralizationTestMode(Enum):
             "SELECT * FROM foo",
             QueryGeneralizationTestMode.BOTH,
         ),
+        (
+            "SELECT a\n -- comment--\n,b --another comment\n FROM books",
+            "redshift",
+            "SELECT a, b FROM books",
+            QueryGeneralizationTestMode.BOTH,
+        ),
         # Parameter normalization.
         (
             "UPDATE  \"books\" SET page_count = page_count + 1, author_count = author_count + 1 WHERE book_title = 'My New Book'",
@@ -107,6 +111,21 @@ class QueryGeneralizationTestMode(Enum):
             "INSERT INTO MyTable (Column1, Column2, Column3) VALUES (?)",
             QueryGeneralizationTestMode.BOTH,
         ),
+        (
+            # Uneven spacing within the IN clause.
+            "SELECT * FROM books WHERE zip_code IN (123,345, 423 )",
+            "redshift",
+            "SELECT * FROM books WHERE zip_code IN (?)",
+            QueryGeneralizationTestMode.BOTH,
+        ),
+        # Uneven spacing in the column list.
+        # This isn't perfect e.g. we still have issues with function calls inside selects.
+        (
+            "SELECT a\n  ,b FROM books",
+            "redshift",
+            "SELECT a, b FROM books",
+            QueryGeneralizationTestMode.BOTH,
+        ),
         (
             textwrap.dedent(
                 """\
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_tool_meta_extractor.py b/metadata-ingestion/tests/unit/sql_parsing/test_tool_meta_extractor.py
index 6f590b5307146..f6566f007f5e6 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_tool_meta_extractor.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_tool_meta_extractor.py
@@ -1,11 +1,14 @@
 from datahub.configuration.datetimes import parse_absolute_time
 from datahub.metadata.urns import CorpUserUrn
 from datahub.sql_parsing.sql_parsing_aggregator import PreparsedQuery
-from datahub.sql_parsing.tool_meta_extractor import ToolMetaExtractor
+from datahub.sql_parsing.tool_meta_extractor import (
+    ToolMetaExtractor,
+    ToolMetaExtractorReport,
+)
 
 
 def test_extract_mode_metadata() -> None:
-    extractor = ToolMetaExtractor()
+    extractor = ToolMetaExtractor(report=ToolMetaExtractorReport())
     query = """\
 select * from LONG_TAIL_COMPANIONS.ADOPTION.PET_PROFILES
 LIMIT 100
@@ -30,8 +33,42 @@ def test_extract_mode_metadata() -> None:
     assert extractor.report.num_queries_meta_extracted["mode"] == 1
 
 
+def test_extract_looker_metadata() -> None:
+    extractor = ToolMetaExtractor(
+        report=ToolMetaExtractorReport(), looker_user_mapping={"7": "john.doe@xyz.com"}
+    )
+    looker_query = """\
+SELECT
+    all_entities_extended_sibling."ENTITY"  AS "all_entities_extended_sibling.entity_type",
+    COUNT(DISTINCT ( all_entities_extended_sibling."URN" )) AS "all_entities_extended_sibling.distinct_count"
+FROM "PUBLIC"."ALL_ENTITIES"
+     AS all_entities_extended_sibling
+GROUP BY
+    1
+ORDER BY
+    1
+FETCH NEXT 50 ROWS ONLY
+-- Looker Query Context '{"user_id":7,"history_slug":"264797031bc403cf382cbefbe3700849","instance_slug":"32654f2ffadf10b1949d4009e52fc6a4"}'
+"""
+
+    entry = PreparsedQuery(
+        query_id=None,
+        query_text=looker_query,
+        upstreams=[],
+        downstream=None,
+        column_lineage=None,
+        column_usage=None,
+        inferred_schema=None,
+        user=CorpUserUrn("mode"),
+        timestamp=parse_absolute_time("2021-08-01T01:02:03Z"),
+    )
+    assert extractor.extract_bi_metadata(entry)
+    assert entry.user == CorpUserUrn("john.doe")
+    assert extractor.report.num_queries_meta_extracted["looker"] == 1
+
+
 def test_extract_no_metadata() -> None:
-    extractor = ToolMetaExtractor()
+    extractor = ToolMetaExtractor(report=ToolMetaExtractorReport())
     query = """\
 select * from LONG_TAIL_COMPANIONS.ADOPTION.PET_PROFILES
 LIMIT 100
@@ -53,3 +90,4 @@ def test_extract_no_metadata() -> None:
     assert not extractor.extract_bi_metadata(entry)
 
     assert extractor.report.num_queries_meta_extracted["mode"] == 0
+    assert extractor.report.num_queries_meta_extracted["looker"] == 0
diff --git a/metadata-ingestion/tests/unit/test_ldap_state.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_ldap_state.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_ldap_state.py
rename to metadata-ingestion/tests/unit/stateful_ingestion/state/test_ldap_state.py
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py
index be6efd3e121ff..5631ad2c69f94 100644
--- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py
@@ -12,17 +12,11 @@
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StatefulStaleMetadataRemovalConfig,
 )
-from datahub.ingestion.source.state.stateful_ingestion_base import (
-    DynamicTypedStateProviderConfig,
-)
 from datahub.ingestion.source.state.usage_common_state import (
     BaseTimeWindowCheckpointState,
 )
 from datahub.utilities.time import datetime_to_ts_millis
 
-GMS_PORT = 8080
-GMS_SERVER = f"http://localhost:{GMS_PORT}"
-
 
 @pytest.fixture
 def stateful_source(mock_datahub_graph: DataHubGraph) -> Iterable[SnowflakeV2Source]:
@@ -39,13 +33,15 @@ def stateful_source(mock_datahub_graph: DataHubGraph) -> Iterable[SnowflakeV2Sou
         password="TST_PWD",
         stateful_ingestion=StatefulStaleMetadataRemovalConfig(
             enabled=True,
-            state_provider=DynamicTypedStateProviderConfig(
-                type="datahub", config={"datahub_api": {"server": GMS_SERVER}}
-            ),
+            # Uses the graph from the pipeline context.
         ),
     )
 
-    with mock.patch("snowflake.connector.connect"):
+    with mock.patch(
+        "datahub.sql_parsing.sql_parsing_aggregator.ToolMetaExtractor.create",
+    ) as mock_checkpoint, mock.patch("snowflake.connector.connect"):
+        mock_checkpoint.return_value = mock.MagicMock()
+
         yield SnowflakeV2Source(ctx=ctx, config=config)
 
 
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py
index 62a42b954daf8..b04d4b86d2e4b 100644
--- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py
@@ -4,6 +4,7 @@
 
 from datahub.ingestion.source.state.entity_removal_state import (
     compute_percent_entities_changed,
+    filter_ignored_entity_types,
 )
 
 EntList = List[str]
@@ -46,3 +47,19 @@ def test_change_percent(
         new_entities=new_entities, old_entities=old_entities
     )
     assert actual_percent_change == expected_percent_change
+
+
+def test_filter_ignored_entity_types():
+    assert filter_ignored_entity_types(
+        [
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)",
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)",
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)",
+            "urn:li:dataProcessInstance:478810e859f870a54f72c681f41af619",
+            "urn:li:query:query1",
+        ]
+    ) == [
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)",
+    ]
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py
index 66564dc856aba..96ab8f7a01a38 100644
--- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py
@@ -10,7 +10,8 @@
 
 from datahub.api.entities.dataprocess.dataprocess_instance import DataProcessInstance
 from datahub.configuration.common import AllowDenyPattern
-from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin
+from datahub.configuration.source_common import DatasetSourceConfigMixin
+from datahub.emitter.mce_builder import DEFAULT_ENV
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py b/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py
index 0e6d60e3440b2..ba40962866f8c 100644
--- a/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py
@@ -14,16 +14,12 @@
 )
 
 # 0. Common client configs.
-datahub_client_configs: Dict[str, Any] = {
-    "full": {
-        "server": "http://localhost:8080",
-        "token": "dummy_test_tok",
-        "timeout_sec": 10,
-        "extra_headers": {},
-        "max_threads": 10,
-    },
-    "simple": {},
-    "default": {},
+datahub_client_full_config = {
+    "server": "http://localhost:8080",
+    "token": "dummy_test_tok",
+    "timeout_sec": 10,
+    "extra_headers": {},
+    "max_threads": 10,
 }
 
 
@@ -41,7 +37,7 @@
     "checkpointing_valid_full_config": (
         DatahubIngestionStateProviderConfig,
         {
-            "datahub_api": datahub_client_configs["full"],
+            "datahub_api": datahub_client_full_config,
         },
         DatahubIngestionStateProviderConfig(
             # This test verifies that the max_threads arg is ignored.
@@ -57,27 +53,14 @@
         ),
         False,
     ),
-    # Simple config
-    "checkpointing_valid_simple_config": (
-        DatahubIngestionStateProviderConfig,
-        {
-            "datahub_api": datahub_client_configs["simple"],
-        },
-        DatahubIngestionStateProviderConfig(
-            datahub_api=DatahubClientConfig(
-                server="http://localhost:8080",
-            ),
-        ),
-        False,
-    ),
     # Default
     "checkpointing_default": (
         DatahubIngestionStateProviderConfig,
         {
-            "datahub_api": datahub_client_configs["default"],
+            "datahub_api": None,
         },
         DatahubIngestionStateProviderConfig(
-            datahub_api=DatahubClientConfig(),
+            datahub_api=None,
         ),
         False,
     ),
@@ -102,7 +85,7 @@
             "max_checkpoint_state_size": 1024,
             "state_provider": {
                 "type": "datahub",
-                "config": datahub_client_configs["full"],
+                "config": datahub_client_full_config,
             },
             "ignore_old_state": True,
             "ignore_new_state": True,
@@ -114,7 +97,7 @@
             ignore_new_state=True,
             state_provider=DynamicTypedStateProviderConfig(
                 type="datahub",
-                config=datahub_client_configs["full"],
+                config=datahub_client_full_config,
             ),
         ),
         False,
diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py
index 875cf3800daf8..f8b6220d18273 100644
--- a/metadata-ingestion/tests/unit/test_athena_source.py
+++ b/metadata-ingestion/tests/unit/test_athena_source.py
@@ -93,7 +93,8 @@ def test_athena_get_table_properties():
             "CreateTime": datetime.now(),
             "LastAccessTime": datetime.now(),
             "PartitionKeys": [
-                {"Name": "testKey", "Type": "string", "Comment": "testComment"}
+                {"Name": "year", "Type": "string", "Comment": "testComment"},
+                {"Name": "month", "Type": "string", "Comment": "testComment"},
             ],
             "Parameters": {
                 "comment": "testComment",
@@ -112,8 +113,18 @@ def test_athena_get_table_properties():
         response=table_metadata
     )
 
+    # Mock partition query results
+    mock_cursor.execute.return_value.description = [
+        ["year"],
+        ["month"],
+    ]
+    mock_cursor.execute.return_value.__iter__.return_value = [["2023", "12"]]
+
     ctx = PipelineContext(run_id="test")
     source = AthenaSource(config=config, ctx=ctx)
+    source.cursor = mock_cursor
+
+    # Test table properties
     description, custom_properties, location = source.get_table_properties(
         inspector=mock_inspector, table=table, schema=schema
     )
@@ -124,13 +135,35 @@ def test_athena_get_table_properties():
         "last_access_time": "2020-04-14 07:00:00",
         "location": "s3://testLocation",
         "outputformat": "testOutputFormat",
-        "partition_keys": '[{"name": "testKey", "type": "string", "comment": "testComment"}]',
+        "partition_keys": '[{"name": "year", "type": "string", "comment": "testComment"}, {"name": "month", "type": "string", "comment": "testComment"}]',
         "serde.serialization.lib": "testSerde",
         "table_type": "testType",
     }
-
     assert location == make_s3_urn("s3://testLocation", "PROD")
 
+    # Test partition functionality
+    partitions = source.get_partitions(
+        inspector=mock_inspector, schema=schema, table=table
+    )
+    assert partitions == ["year", "month"]
+
+    # Verify the correct SQL query was generated for partitions
+    expected_query = """\
+select year,month from "test_schema"."test_table$partitions" \
+where CAST(year as VARCHAR) || '-' || CAST(month as VARCHAR) = \
+(select max(CAST(year as VARCHAR) || '-' || CAST(month as VARCHAR)) \
+from "test_schema"."test_table$partitions")"""
+    mock_cursor.execute.assert_called_once()
+    actual_query = mock_cursor.execute.call_args[0][0]
+    assert actual_query == expected_query
+
+    # Verify partition cache was populated correctly
+    assert source.table_partition_cache[schema][table].partitions == partitions
+    assert source.table_partition_cache[schema][table].max_partition == {
+        "year": "2023",
+        "month": "12",
+    }
+
 
 def test_get_column_type_simple_types():
     assert isinstance(
@@ -214,3 +247,9 @@ def test_column_type_complex_combination():
     assert isinstance(
         result._STRUCT_fields[2][1].item_type._STRUCT_fields[1][1], types.String
     )
+
+
+def test_casted_partition_key():
+    from datahub.ingestion.source.sql.athena import AthenaSource
+
+    assert AthenaSource._casted_partition_key("test_col") == "CAST(test_col as VARCHAR)"
diff --git a/metadata-ingestion/tests/unit/test_aws_common.py b/metadata-ingestion/tests/unit/test_aws_common.py
new file mode 100644
index 0000000000000..9291fb91134b1
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_aws_common.py
@@ -0,0 +1,328 @@
+import json
+import os
+from unittest.mock import MagicMock, patch
+
+import boto3
+import pytest
+from moto import mock_iam, mock_lambda, mock_sts
+
+from datahub.ingestion.source.aws.aws_common import (
+    AwsConnectionConfig,
+    AwsEnvironment,
+    detect_aws_environment,
+    get_current_identity,
+    get_instance_metadata_token,
+    get_instance_role_arn,
+    is_running_on_ec2,
+)
+
+
+@pytest.fixture
+def mock_aws_config():
+    return AwsConnectionConfig(
+        aws_access_key_id="test-key",
+        aws_secret_access_key="test-secret",
+        aws_region="us-east-1",
+    )
+
+
+class TestAwsCommon:
+    def test_environment_detection_no_environment(self):
+        """Test environment detection when no AWS environment is present"""
+        with patch.dict(os.environ, {}, clear=True):
+            assert detect_aws_environment() == AwsEnvironment.UNKNOWN
+
+    def test_environment_detection_lambda(self):
+        """Test Lambda environment detection"""
+        with patch.dict(os.environ, {"AWS_LAMBDA_FUNCTION_NAME": "test-function"}):
+            assert detect_aws_environment() == AwsEnvironment.LAMBDA
+
+    def test_environment_detection_lambda_cloudformation(self):
+        """Test CloudFormation Lambda environment detection"""
+        with patch.dict(
+            os.environ,
+            {
+                "AWS_LAMBDA_FUNCTION_NAME": "test-function",
+                "AWS_EXECUTION_ENV": "CloudFormation.xxx",
+            },
+        ):
+            assert detect_aws_environment() == AwsEnvironment.CLOUD_FORMATION
+
+    def test_environment_detection_eks(self):
+        """Test EKS environment detection"""
+        with patch.dict(
+            os.environ,
+            {
+                "AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token",
+                "AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/test-role",
+            },
+        ):
+            assert detect_aws_environment() == AwsEnvironment.EKS
+
+    def test_environment_detection_app_runner(self):
+        """Test App Runner environment detection"""
+        with patch.dict(os.environ, {"AWS_APP_RUNNER_SERVICE_ID": "service-id"}):
+            assert detect_aws_environment() == AwsEnvironment.APP_RUNNER
+
+    def test_environment_detection_ecs(self):
+        """Test ECS environment detection"""
+        with patch.dict(
+            os.environ, {"ECS_CONTAINER_METADATA_URI_V4": "http://169.254.170.2/v4"}
+        ):
+            assert detect_aws_environment() == AwsEnvironment.ECS
+
+    def test_environment_detection_beanstalk(self):
+        """Test Elastic Beanstalk environment detection"""
+        with patch.dict(os.environ, {"ELASTIC_BEANSTALK_ENVIRONMENT_NAME": "my-env"}):
+            assert detect_aws_environment() == AwsEnvironment.BEANSTALK
+
+    @patch("requests.put")
+    def test_ec2_metadata_token(self, mock_put):
+        """Test EC2 metadata token retrieval"""
+        mock_put.return_value.status_code = 200
+        mock_put.return_value.text = "token123"
+
+        token = get_instance_metadata_token()
+        assert token == "token123"
+
+        mock_put.assert_called_once_with(
+            "http://169.254.169.254/latest/api/token",
+            headers={"X-aws-ec2-metadata-token-ttl-seconds": "21600"},
+            timeout=1,
+        )
+
+    @patch("requests.put")
+    def test_ec2_metadata_token_failure(self, mock_put):
+        """Test EC2 metadata token failure case"""
+        mock_put.return_value.status_code = 404
+
+        token = get_instance_metadata_token()
+        assert token is None
+
+    @patch("requests.get")
+    @patch("requests.put")
+    def test_is_running_on_ec2(self, mock_put, mock_get):
+        """Test EC2 instance detection with IMDSv2"""
+        mock_put.return_value.status_code = 200
+        mock_put.return_value.text = "token123"
+        mock_get.return_value.status_code = 200
+
+        assert is_running_on_ec2() is True
+
+        mock_put.assert_called_once_with(
+            "http://169.254.169.254/latest/api/token",
+            headers={"X-aws-ec2-metadata-token-ttl-seconds": "21600"},
+            timeout=1,
+        )
+        mock_get.assert_called_once_with(
+            "http://169.254.169.254/latest/meta-data/instance-id",
+            headers={"X-aws-ec2-metadata-token": "token123"},
+            timeout=1,
+        )
+
+    @patch("requests.get")
+    @patch("requests.put")
+    def test_is_running_on_ec2_failure(self, mock_put, mock_get):
+        """Test EC2 instance detection failure"""
+        mock_put.return_value.status_code = 404
+        assert is_running_on_ec2() is False
+
+        mock_put.return_value.status_code = 200
+        mock_put.return_value.text = "token123"
+        mock_get.return_value.status_code = 404
+        assert is_running_on_ec2() is False
+
+    @mock_sts
+    @mock_lambda
+    @mock_iam
+    def test_get_current_identity_lambda(self):
+        """Test getting identity in Lambda environment"""
+        with patch.dict(
+            os.environ,
+            {
+                "AWS_LAMBDA_FUNCTION_NAME": "test-function",
+                "AWS_DEFAULT_REGION": "us-east-1",
+            },
+        ):
+            # Create IAM role first with proper trust policy
+            iam_client = boto3.client("iam", region_name="us-east-1")
+            trust_policy = {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Principal": {"Service": "lambda.amazonaws.com"},
+                        "Action": "sts:AssumeRole",
+                    }
+                ],
+            }
+            iam_client.create_role(
+                RoleName="test-role", AssumeRolePolicyDocument=json.dumps(trust_policy)
+            )
+
+            lambda_client = boto3.client("lambda", region_name="us-east-1")
+            lambda_client.create_function(
+                FunctionName="test-function",
+                Runtime="python3.8",
+                Role="arn:aws:iam::123456789012:role/test-role",
+                Handler="index.handler",
+                Code={"ZipFile": b"def handler(event, context): pass"},
+            )
+
+            role_arn, source = get_current_identity()
+            assert source == "lambda.amazonaws.com"
+            assert role_arn == "arn:aws:iam::123456789012:role/test-role"
+
+    @patch("requests.get")
+    @patch("requests.put")
+    @mock_sts
+    def test_get_instance_role_arn_success(self, mock_put, mock_get):
+        """Test getting EC2 instance role ARN"""
+        mock_put.return_value.status_code = 200
+        mock_put.return_value.text = "token123"
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.text = "test-role"
+
+        with patch("boto3.client") as mock_boto:
+            mock_sts = MagicMock()
+            mock_sts.get_caller_identity.return_value = {
+                "Arn": "arn:aws:sts::123456789012:assumed-role/test-role/instance"
+            }
+            mock_boto.return_value = mock_sts
+
+            role_arn = get_instance_role_arn()
+            assert (
+                role_arn == "arn:aws:sts::123456789012:assumed-role/test-role/instance"
+            )
+
+    @mock_sts
+    def test_aws_connection_config_basic(self, mock_aws_config):
+        """Test basic AWS connection configuration"""
+        session = mock_aws_config.get_session()
+        creds = session.get_credentials()
+        assert creds.access_key == "test-key"
+        assert creds.secret_key == "test-secret"
+
+    @mock_sts
+    def test_aws_connection_config_with_session_token(self):
+        """Test AWS connection with session token"""
+        config = AwsConnectionConfig(
+            aws_access_key_id="test-key",
+            aws_secret_access_key="test-secret",
+            aws_session_token="test-token",
+            aws_region="us-east-1",
+        )
+
+        session = config.get_session()
+        creds = session.get_credentials()
+        assert creds.token == "test-token"
+
+    @mock_sts
+    def test_aws_connection_config_role_assumption(self):
+        """Test AWS connection with role assumption"""
+        config = AwsConnectionConfig(
+            aws_access_key_id="test-key",
+            aws_secret_access_key="test-secret",
+            aws_region="us-east-1",
+            aws_role="arn:aws:iam::123456789012:role/test-role",
+        )
+
+        with patch(
+            "datahub.ingestion.source.aws.aws_common.get_current_identity"
+        ) as mock_identity:
+            mock_identity.return_value = (None, None)
+            session = config.get_session()
+            creds = session.get_credentials()
+            assert creds is not None
+
+    @mock_sts
+    def test_aws_connection_config_skip_role_assumption(self):
+        """Test AWS connection skipping role assumption when already in role"""
+        config = AwsConnectionConfig(
+            aws_region="us-east-1",
+            aws_role="arn:aws:iam::123456789012:role/current-role",
+        )
+
+        with patch(
+            "datahub.ingestion.source.aws.aws_common.get_current_identity"
+        ) as mock_identity:
+            mock_identity.return_value = (
+                "arn:aws:iam::123456789012:role/current-role",
+                "ec2.amazonaws.com",
+            )
+            session = config.get_session()
+            assert session is not None
+
+    @mock_sts
+    def test_aws_connection_config_multiple_roles(self):
+        """Test AWS connection with multiple role assumption"""
+        config = AwsConnectionConfig(
+            aws_access_key_id="test-key",
+            aws_secret_access_key="test-secret",
+            aws_region="us-east-1",
+            aws_role=[
+                "arn:aws:iam::123456789012:role/role1",
+                "arn:aws:iam::123456789012:role/role2",
+            ],
+        )
+
+        with patch(
+            "datahub.ingestion.source.aws.aws_common.get_current_identity"
+        ) as mock_identity:
+            mock_identity.return_value = (None, None)
+            session = config.get_session()
+            assert session is not None
+
+    def test_aws_connection_config_validation_error(self):
+        """Test AWS connection validation"""
+        with patch.dict(
+            "os.environ",
+            {
+                "AWS_ACCESS_KEY_ID": "test-key",
+                # Deliberately missing AWS_SECRET_ACCESS_KEY
+                "AWS_DEFAULT_REGION": "us-east-1",
+            },
+            clear=True,
+        ):
+            config = AwsConnectionConfig()  # Let it pick up from environment
+            session = config.get_session()
+            with pytest.raises(
+                Exception,
+                match="Partial credentials found in env, missing: AWS_SECRET_ACCESS_KEY",
+            ):
+                session.get_credentials()
+
+    @pytest.mark.parametrize(
+        "env_vars,expected_environment",
+        [
+            ({}, AwsEnvironment.UNKNOWN),
+            ({"AWS_LAMBDA_FUNCTION_NAME": "test"}, AwsEnvironment.LAMBDA),
+            (
+                {
+                    "AWS_LAMBDA_FUNCTION_NAME": "test",
+                    "AWS_EXECUTION_ENV": "CloudFormation",
+                },
+                AwsEnvironment.CLOUD_FORMATION,
+            ),
+            (
+                {
+                    "AWS_WEB_IDENTITY_TOKEN_FILE": "/token",
+                    "AWS_ROLE_ARN": "arn:aws:iam::123:role/test",
+                },
+                AwsEnvironment.EKS,
+            ),
+            ({"AWS_APP_RUNNER_SERVICE_ID": "service-123"}, AwsEnvironment.APP_RUNNER),
+            (
+                {"ECS_CONTAINER_METADATA_URI_V4": "http://169.254.170.2"},
+                AwsEnvironment.ECS,
+            ),
+            (
+                {"ELASTIC_BEANSTALK_ENVIRONMENT_NAME": "my-env"},
+                AwsEnvironment.BEANSTALK,
+            ),
+        ],
+    )
+    def test_environment_detection_parametrized(self, env_vars, expected_environment):
+        """Parametrized test for environment detection with different configurations"""
+        with patch.dict(os.environ, env_vars, clear=True):
+            assert detect_aws_environment() == expected_environment
diff --git a/metadata-ingestion/tests/unit/test_bigquery_lineage.py b/metadata-ingestion/tests/unit/test_bigquery_lineage.py
deleted file mode 100644
index 7456f2fd1d91c..0000000000000
--- a/metadata-ingestion/tests/unit/test_bigquery_lineage.py
+++ /dev/null
@@ -1,137 +0,0 @@
-import datetime
-from typing import Dict, List, Set
-
-import pytest
-
-from datahub.ingestion.source.bigquery_v2.bigquery_audit import (
-    BigQueryTableRef,
-    QueryEvent,
-)
-from datahub.ingestion.source.bigquery_v2.bigquery_config import BigQueryV2Config
-from datahub.ingestion.source.bigquery_v2.bigquery_report import BigQueryV2Report
-from datahub.ingestion.source.bigquery_v2.common import BigQueryIdentifierBuilder
-from datahub.ingestion.source.bigquery_v2.lineage import (
-    BigqueryLineageExtractor,
-    LineageEdge,
-)
-from datahub.sql_parsing.schema_resolver import SchemaResolver
-
-
-@pytest.fixture
-def lineage_entries() -> List[QueryEvent]:
-    return [
-        QueryEvent(
-            timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
-            actor_email="bla@bla.com",
-            query="""
-                INSERT INTO `my_project.my_dataset.my_table`
-                SELECT first.a, second.b FROM `my_project.my_dataset.my_source_table1` first
-                LEFT JOIN `my_project.my_dataset.my_source_table2` second ON first.id = second.id
-            """,
-            statementType="INSERT",
-            project_id="proj_12344",
-            end_time=None,
-            referencedTables=[
-                BigQueryTableRef.from_string_name(
-                    "projects/my_project/datasets/my_dataset/tables/my_source_table1"
-                ),
-                BigQueryTableRef.from_string_name(
-                    "projects/my_project/datasets/my_dataset/tables/my_source_table2"
-                ),
-            ],
-            destinationTable=BigQueryTableRef.from_string_name(
-                "projects/my_project/datasets/my_dataset/tables/my_table"
-            ),
-        ),
-        QueryEvent(
-            timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
-            actor_email="bla@bla.com",
-            query="testQuery",
-            statementType="SELECT",
-            project_id="proj_12344",
-            end_time=datetime.datetime.fromtimestamp(
-                1617295943.17321, tz=datetime.timezone.utc
-            ),
-            referencedTables=[
-                BigQueryTableRef.from_string_name(
-                    "projects/my_project/datasets/my_dataset/tables/my_source_table3"
-                ),
-            ],
-            destinationTable=BigQueryTableRef.from_string_name(
-                "projects/my_project/datasets/my_dataset/tables/my_table"
-            ),
-        ),
-        QueryEvent(
-            timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
-            actor_email="bla@bla.com",
-            query="testQuery",
-            statementType="SELECT",
-            project_id="proj_12344",
-            referencedViews=[
-                BigQueryTableRef.from_string_name(
-                    "projects/my_project/datasets/my_dataset/tables/my_source_view1"
-                ),
-            ],
-            destinationTable=BigQueryTableRef.from_string_name(
-                "projects/my_project/datasets/my_dataset/tables/my_table"
-            ),
-        ),
-    ]
-
-
-def test_lineage_with_timestamps(lineage_entries: List[QueryEvent]) -> None:
-    config = BigQueryV2Config()
-    report = BigQueryV2Report()
-    extractor: BigqueryLineageExtractor = BigqueryLineageExtractor(
-        config,
-        report,
-        schema_resolver=SchemaResolver(platform="bigquery"),
-        identifiers=BigQueryIdentifierBuilder(config, report),
-    )
-
-    bq_table = BigQueryTableRef.from_string_name(
-        "projects/my_project/datasets/my_dataset/tables/my_table"
-    )
-
-    lineage_map: Dict[str, Set[LineageEdge]] = extractor._create_lineage_map(
-        iter(lineage_entries)
-    )
-
-    upstream_lineage = extractor.get_lineage_for_table(
-        bq_table=bq_table,
-        bq_table_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)",
-        lineage_metadata=lineage_map,
-    )
-    assert upstream_lineage
-    assert len(upstream_lineage.upstreams) == 4
-
-
-def test_column_level_lineage(lineage_entries: List[QueryEvent]) -> None:
-    config = BigQueryV2Config(extract_column_lineage=True, incremental_lineage=False)
-    report = BigQueryV2Report()
-    extractor: BigqueryLineageExtractor = BigqueryLineageExtractor(
-        config,
-        report,
-        schema_resolver=SchemaResolver(platform="bigquery"),
-        identifiers=BigQueryIdentifierBuilder(config, report),
-    )
-
-    bq_table = BigQueryTableRef.from_string_name(
-        "projects/my_project/datasets/my_dataset/tables/my_table"
-    )
-
-    lineage_map: Dict[str, Set[LineageEdge]] = extractor._create_lineage_map(
-        lineage_entries[:1],
-    )
-
-    upstream_lineage = extractor.get_lineage_for_table(
-        bq_table=bq_table,
-        bq_table_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)",
-        lineage_metadata=lineage_map,
-    )
-    assert upstream_lineage
-    assert len(upstream_lineage.upstreams) == 2
-    assert (
-        upstream_lineage.fineGrainedLineages
-        and len(upstream_lineage.fineGrainedLineages) == 2
-    )
diff --git a/metadata-ingestion/tests/unit/test_cassandra_source.py b/metadata-ingestion/tests/unit/test_cassandra_source.py
new file mode 100644
index 0000000000000..75dedde76c7c8
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_cassandra_source.py
@@ -0,0 +1,80 @@
+import json
+import logging
+import re
+from typing import Any, Dict, List, Tuple
+
+import pytest
+
+from datahub.ingestion.source.cassandra.cassandra import CassandraToSchemaFieldConverter
+from datahub.ingestion.source.cassandra.cassandra_api import CassandraColumn
+from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField
+
+logger = logging.getLogger(__name__)
+
+
+def assert_field_paths_are_unique(fields: List[SchemaField]) -> None:
+    fields_paths = [f.fieldPath for f in fields if re.match(".*[^]]$", f.fieldPath)]
+
+    if fields_paths:
+        assert len(fields_paths) == len(set(fields_paths))
+
+
+def assert_field_paths_match(
+    fields: List[SchemaField], expected_field_paths: List[str]
+) -> None:
+    logger.debug('FieldPaths=\n"' + '",\n"'.join(f.fieldPath for f in fields) + '"')
+    assert len(fields) == len(expected_field_paths)
+    for f, efp in zip(fields, expected_field_paths):
+        assert f.fieldPath == efp
+    assert_field_paths_are_unique(fields)
+
+
+# TODO: cover one for every item on https://cassandra.apache.org/doc/stable/cassandra/cql/types.html (version 4.1)
+schema_test_cases: Dict[str, Tuple[str, List[str]]] = {
+    "all_types_on_4.1": (
+        """{
+            "column_infos": [
+                {"keyspace_name": "playground", "table_name": "people", "column_name": "birthday", "clustering_order": "none", "column_name_bytes": null, "kind": "regular", "position": -1, "type": "timestamp"},
+                {"keyspace_name": "playground", "table_name": "people", "column_name": "email", "clustering_order": "none", "column_name_bytes": null, "kind": "partition_key", "position": 0, "type": "text"},
+                {"keyspace_name": "playground", "table_name": "people", "column_name": "name", "clustering_order": "none", "column_name_bytes": null, "kind": "regular", "position": -1, "type": "text"}
+            ]
+        }""",
+        [
+            "birthday",
+            "email",
+            "name",
+        ],
+    )
+}
+
+
+@pytest.mark.parametrize(
+    "schema, expected_field_paths",
+    schema_test_cases.values(),
+    ids=schema_test_cases.keys(),
+)
+def test_cassandra_schema_conversion(
+    schema: str, expected_field_paths: List[str]
+) -> None:
+    schema_dict: Dict[str, List[Any]] = json.loads(schema)
+    column_infos: List = schema_dict["column_infos"]
+
+    column_list: List[CassandraColumn] = [
+        CassandraColumn(
+            keyspace_name=row["keyspace_name"],
+            table_name=row["table_name"],
+            column_name=row["column_name"],
+            clustering_order=row["clustering_order"],
+            kind=row["kind"],
+            position=row["position"],
+            type=row["type"],
+        )
+        for row in column_infos
+    ]
+    actual_fields = list(CassandraToSchemaFieldConverter.get_schema_fields(column_list))
+    assert_field_paths_match(actual_fields, expected_field_paths)
+
+
+def test_no_properties_in_mappings_schema() -> None:
+    fields = list(CassandraToSchemaFieldConverter.get_schema_fields([]))
+    assert fields == []
diff --git a/metadata-ingestion/tests/unit/test_confluent_schema_registry.py b/metadata-ingestion/tests/unit/test_confluent_schema_registry.py
index b047cd16c52a9..3500636f00edd 100644
--- a/metadata-ingestion/tests/unit/test_confluent_schema_registry.py
+++ b/metadata-ingestion/tests/unit/test_confluent_schema_registry.py
@@ -8,7 +8,7 @@
 )
 
 from datahub.ingestion.source.confluent_schema_registry import ConfluentSchemaRegistry
-from datahub.ingestion.source.kafka import KafkaSourceConfig, KafkaSourceReport
+from datahub.ingestion.source.kafka.kafka import KafkaSourceConfig, KafkaSourceReport
 
 
 class ConfluentSchemaRegistryTest(unittest.TestCase):
diff --git a/metadata-ingestion/tests/unit/test_datahub_source.py b/metadata-ingestion/tests/unit/test_datahub_source.py
index adc131362b326..67b2b85d9af6d 100644
--- a/metadata-ingestion/tests/unit/test_datahub_source.py
+++ b/metadata-ingestion/tests/unit/test_datahub_source.py
@@ -1,51 +1,41 @@
-from dataclasses import dataclass
+from typing import Any, Dict
 
 import pytest
 
-from datahub.ingestion.source.datahub.datahub_database_reader import (
-    VersionOrderable,
-    VersionOrderer,
-)
-
-
-@dataclass
-class MockRow(VersionOrderable):
-    createdon: int
-    version: int
-    urn: str
+from datahub.ingestion.source.datahub.datahub_database_reader import VersionOrderer
 
 
 @pytest.fixture
 def rows():
     return [
-        MockRow(0, 0, "one"),
-        MockRow(0, 1, "one"),
-        MockRow(0, 0, "two"),
-        MockRow(0, 0, "three"),
-        MockRow(0, 1, "three"),
-        MockRow(0, 2, "three"),
-        MockRow(0, 1, "two"),
-        MockRow(0, 4, "three"),
-        MockRow(0, 5, "three"),
-        MockRow(1, 6, "three"),
-        MockRow(1, 0, "four"),
-        MockRow(2, 0, "five"),
-        MockRow(2, 1, "six"),
-        MockRow(2, 0, "six"),
-        MockRow(3, 0, "seven"),
-        MockRow(3, 0, "eight"),
+        {"createdon": 0, "version": 0, "urn": "one"},
+        {"createdon": 0, "version": 1, "urn": "one"},
+        {"createdon": 0, "version": 0, "urn": "two"},
+        {"createdon": 0, "version": 0, "urn": "three"},
+        {"createdon": 0, "version": 1, "urn": "three"},
+        {"createdon": 0, "version": 2, "urn": "three"},
+        {"createdon": 0, "version": 1, "urn": "two"},
+        {"createdon": 0, "version": 4, "urn": "three"},
+        {"createdon": 0, "version": 5, "urn": "three"},
+        {"createdon": 1, "version": 6, "urn": "three"},
+        {"createdon": 1, "version": 0, "urn": "four"},
+        {"createdon": 2, "version": 0, "urn": "five"},
+        {"createdon": 2, "version": 1, "urn": "six"},
+        {"createdon": 2, "version": 0, "urn": "six"},
+        {"createdon": 3, "version": 0, "urn": "seven"},
+        {"createdon": 3, "version": 0, "urn": "eight"},
     ]
 
 
 def test_version_orderer(rows):
-    orderer = VersionOrderer[MockRow](enabled=True)
+    orderer = VersionOrderer[Dict[str, Any]](enabled=True)
     ordered_rows = list(orderer(rows))
     assert ordered_rows == sorted(
-        ordered_rows, key=lambda x: (x.createdon, x.version == 0)
+        ordered_rows, key=lambda x: (x["createdon"], x["version"] == 0)
     )
 
 
 def test_version_orderer_disabled(rows):
-    orderer = VersionOrderer[MockRow](enabled=False)
+    orderer = VersionOrderer[Dict[str, Any]](enabled=False)
     ordered_rows = list(orderer(rows))
     assert ordered_rows == rows
diff --git a/metadata-ingestion/tests/unit/test_dbt_source.py b/metadata-ingestion/tests/unit/test_dbt_source.py
index 7d01ecd034523..0a86929783701 100644
--- a/metadata-ingestion/tests/unit/test_dbt_source.py
+++ b/metadata-ingestion/tests/unit/test_dbt_source.py
@@ -1,4 +1,3 @@
-import doctest
 from datetime import timedelta
 from typing import Dict, List, Union
 from unittest import mock
@@ -10,6 +9,7 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.dbt import dbt_cloud
 from datahub.ingestion.source.dbt.dbt_cloud import DBTCloudConfig
+from datahub.ingestion.source.dbt.dbt_common import DBTNode
 from datahub.ingestion.source.dbt.dbt_core import (
     DBTCoreConfig,
     DBTCoreSource,
@@ -21,6 +21,7 @@
     OwnershipSourceTypeClass,
     OwnershipTypeClass,
 )
+from datahub.testing.doctest import assert_doctest
 
 
 def create_owners_list_from_urn_list(
@@ -253,6 +254,47 @@ def test_dbt_config_prefer_sql_parser_lineage():
     assert config.prefer_sql_parser_lineage is True
 
 
+def test_dbt_prefer_sql_parser_lineage_no_self_reference():
+    ctx = PipelineContext(run_id="test-run-id")
+    config = DBTCoreConfig.parse_obj(
+        {
+            **create_base_dbt_config(),
+            "skip_sources_in_lineage": True,
+            "prefer_sql_parser_lineage": True,
+        }
+    )
+    source: DBTCoreSource = DBTCoreSource(config, ctx, "dbt")
+    all_nodes_map = {
+        "model1": DBTNode(
+            name="model1",
+            database=None,
+            schema=None,
+            alias=None,
+            comment="",
+            description="",
+            language=None,
+            raw_code=None,
+            dbt_adapter="postgres",
+            dbt_name="model1",
+            dbt_file_path=None,
+            dbt_package_name=None,
+            node_type="model",
+            materialization="table",
+            max_loaded_at=None,
+            catalog_type=None,
+            missing_from_catalog=False,
+            owner=None,
+            compiled_code="SELECT d FROM results WHERE d > (SELECT MAX(d) FROM model1)",
+        ),
+    }
+    source._infer_schemas_and_update_cll(all_nodes_map)
+    upstream_lineage = source._create_lineage_aspect_for_dbt_node(
+        all_nodes_map["model1"], all_nodes_map
+    )
+    assert upstream_lineage is not None
+    assert len(upstream_lineage.upstreams) == 1
+
+
 def test_dbt_s3_config():
     # test missing aws config
     config_dict: dict = {
@@ -400,7 +442,7 @@ def test_dbt_cloud_config_with_defined_metadata_endpoint():
 
 
 def test_infer_metadata_endpoint() -> None:
-    assert doctest.testmod(dbt_cloud, raise_on_error=True).attempted > 0
+    assert_doctest(dbt_cloud)
 
 
 def test_dbt_time_parsing() -> None:
diff --git a/metadata-ingestion/tests/unit/test_gc.py b/metadata-ingestion/tests/unit/test_gc.py
new file mode 100644
index 0000000000000..8f00d5e064db8
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_gc.py
@@ -0,0 +1,157 @@
+import unittest
+from datetime import datetime, timezone
+from unittest.mock import MagicMock, patch
+
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.source.gc.dataprocess_cleanup import (
+    DataJobEntity,
+    DataProcessCleanup,
+    DataProcessCleanupConfig,
+    DataProcessCleanupReport,
+)
+
+
+class TestDataProcessCleanup(unittest.TestCase):
+    def setUp(self):
+        self.ctx = PipelineContext(run_id="test_run")
+        self.ctx.graph = MagicMock()
+        self.config = DataProcessCleanupConfig()
+        self.report = DataProcessCleanupReport()
+        self.cleanup = DataProcessCleanup(
+            self.ctx, self.config, self.report, dry_run=True
+        )
+
+    @patch(
+        "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis"
+    )
+    def test_delete_dpi_from_datajobs(self, mock_fetch_dpis):
+        job = DataJobEntity(
+            urn="urn:li:dataJob:1",
+            flow_urn="urn:li:dataFlow:1",
+            lastIngested=int(datetime.now(timezone.utc).timestamp()),
+            jobId="job1",
+            dataPlatformInstance="urn:li:dataPlatformInstance:1",
+            total_runs=10,
+        )
+        mock_fetch_dpis.return_value = [
+            {
+                "urn": f"urn:li:dataprocessInstance:{i}",
+                "created": {
+                    "time": int(datetime.now(timezone.utc).timestamp() + i) * 1000
+                },
+            }
+            for i in range(10)
+        ]
+        self.cleanup.delete_dpi_from_datajobs(job)
+        self.assertEqual(5, self.report.num_aspects_removed)
+
+    @patch(
+        "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis"
+    )
+    def test_delete_dpi_from_datajobs_without_dpis(self, mock_fetch_dpis):
+        job = DataJobEntity(
+            urn="urn:li:dataJob:1",
+            flow_urn="urn:li:dataFlow:1",
+            lastIngested=int(datetime.now(timezone.utc).timestamp()),
+            jobId="job1",
+            dataPlatformInstance="urn:li:dataPlatformInstance:1",
+            total_runs=10,
+        )
+        mock_fetch_dpis.return_value = []
+        self.cleanup.delete_dpi_from_datajobs(job)
+        self.assertEqual(0, self.report.num_aspects_removed)
+
+    @patch(
+        "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis"
+    )
+    def test_delete_dpi_from_datajobs_without_dpi_created_time(self, mock_fetch_dpis):
+        job = DataJobEntity(
+            urn="urn:li:dataJob:1",
+            flow_urn="urn:li:dataFlow:1",
+            lastIngested=int(datetime.now(timezone.utc).timestamp()),
+            jobId="job1",
+            dataPlatformInstance="urn:li:dataPlatformInstance:1",
+            total_runs=10,
+        )
+        mock_fetch_dpis.return_value = [
+            {"urn": f"urn:li:dataprocessInstance:{i}"} for i in range(10)
+        ] + [
+            {
+                "urn": "urn:li:dataprocessInstance:11",
+                "created": {"time": int(datetime.now(timezone.utc).timestamp() * 1000)},
+            }
+        ]
+        self.cleanup.delete_dpi_from_datajobs(job)
+        self.assertEqual(10, self.report.num_aspects_removed)
+
+    @patch(
+        "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis"
+    )
+    def test_delete_dpi_from_datajobs_without_dpi_null_created_time(
+        self, mock_fetch_dpis
+    ):
+        job = DataJobEntity(
+            urn="urn:li:dataJob:1",
+            flow_urn="urn:li:dataFlow:1",
+            lastIngested=int(datetime.now(timezone.utc).timestamp()),
+            jobId="job1",
+            dataPlatformInstance="urn:li:dataPlatformInstance:1",
+            total_runs=10,
+        )
+        mock_fetch_dpis.return_value = [
+            {"urn": f"urn:li:dataprocessInstance:{i}"} for i in range(10)
+        ] + [
+            {
+                "urn": "urn:li:dataprocessInstance:11",
+                "created": {"time": None},
+            }
+        ]
+        self.cleanup.delete_dpi_from_datajobs(job)
+        self.assertEqual(11, self.report.num_aspects_removed)
+
+    @patch(
+        "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis"
+    )
+    def test_delete_dpi_from_datajobs_without_dpi_without_time(self, mock_fetch_dpis):
+        job = DataJobEntity(
+            urn="urn:li:dataJob:1",
+            flow_urn="urn:li:dataFlow:1",
+            lastIngested=int(datetime.now(timezone.utc).timestamp()),
+            jobId="job1",
+            dataPlatformInstance="urn:li:dataPlatformInstance:1",
+            total_runs=10,
+        )
+        mock_fetch_dpis.return_value = [
+            {"urn": f"urn:li:dataprocessInstance:{i}"} for i in range(10)
+        ] + [
+            {
+                "urn": "urn:li:dataprocessInstance:11",
+                "created": None,
+            }
+        ]
+        self.cleanup.delete_dpi_from_datajobs(job)
+        self.assertEqual(11, self.report.num_aspects_removed)
+
+    def test_fetch_dpis(self):
+        assert self.cleanup.ctx.graph
+        self.cleanup.ctx.graph = MagicMock()
+        self.cleanup.ctx.graph.execute_graphql.return_value = {
+            "dataJob": {
+                "runs": {
+                    "runs": [
+                        {
+                            "urn": "urn:li:dataprocessInstance:1",
+                            "created": {
+                                "time": int(datetime.now(timezone.utc).timestamp())
+                            },
+                        }
+                    ]
+                }
+            }
+        }
+        dpis = self.cleanup.fetch_dpis("urn:li:dataJob:1", 10)
+        self.assertEqual(len(dpis), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py b/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py
index 18b0d9fd40041..52d7aa7f509c9 100644
--- a/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py
+++ b/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py
@@ -51,7 +51,7 @@ def make_mcpw(
 ) -> MetadataChangeProposalWrapper:
     return MetadataChangeProposalWrapper(
         entityUrn=entity_urn,
-        entityType=Urn.create_from_string(entity_urn).get_type(),
+        entityType=Urn.from_string(entity_urn).get_type(),
         aspectName=aspect_name,
         changeType="UPSERT",
         aspect=aspect,
@@ -65,7 +65,7 @@ def make_mcpc(
 ) -> MetadataChangeProposalClass:
     return MetadataChangeProposalClass(
         entityUrn=entity_urn,
-        entityType=Urn.create_from_string(entity_urn).get_type(),
+        entityType=Urn.from_string(entity_urn).get_type(),
         aspectName=aspect_name,
         changeType="UPSERT",
         aspect=aspect,
diff --git a/metadata-ingestion/tests/unit/test_iceberg.py b/metadata-ingestion/tests/unit/test_iceberg.py
index c8c6c6ac8a85d..3afa26b35dfe9 100644
--- a/metadata-ingestion/tests/unit/test_iceberg.py
+++ b/metadata-ingestion/tests/unit/test_iceberg.py
@@ -1,10 +1,23 @@
 import uuid
 from decimal import Decimal
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
+from unittest import TestCase
+from unittest.mock import patch
 
 import pytest
 from pydantic import ValidationError
+from pyiceberg.exceptions import (
+    NoSuchIcebergTableError,
+    NoSuchNamespaceError,
+    NoSuchPropertyException,
+    NoSuchTableError,
+    ServerError,
+)
+from pyiceberg.io.pyarrow import PyArrowFileIO
+from pyiceberg.partitioning import PartitionSpec
 from pyiceberg.schema import Schema
+from pyiceberg.table import Table
+from pyiceberg.table.metadata import TableMetadataV2
 from pyiceberg.types import (
     BinaryType,
     BooleanType,
@@ -28,17 +41,21 @@
     UUIDType,
 )
 
+from datahub.configuration.common import AllowDenyPattern
 from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.iceberg.iceberg import (
     IcebergProfiler,
     IcebergSource,
     IcebergSourceConfig,
 )
+from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.com.linkedin.pegasus2avro.schema import ArrayType, SchemaField
 from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     BooleanTypeClass,
     BytesTypeClass,
+    DatasetSnapshotClass,
     DateTypeClass,
     FixedTypeClass,
     NumberTypeClass,
@@ -48,11 +65,13 @@
 )
 
 
-def with_iceberg_source() -> IcebergSource:
+def with_iceberg_source(processing_threads: int = 1, **kwargs: Any) -> IcebergSource:
     catalog = {"test": {"type": "rest"}}
     return IcebergSource(
         ctx=PipelineContext(run_id="iceberg-source-test"),
-        config=IcebergSourceConfig(catalog=catalog),
+        config=IcebergSourceConfig(
+            catalog=catalog, processing_threads=processing_threads, **kwargs
+        ),
     )
 
 
@@ -515,3 +534,605 @@ def test_avro_decimal_bytes_nullable() -> None:
     print(
         f"After avro parsing, _nullable attribute is preserved:  {boolean_avro_schema}"
     )
+
+
+class MockCatalog:
+    def __init__(self, tables: Dict[str, Dict[str, Callable[[], Table]]]):
+        """
+
+        :param tables: Dictionary containing namespaces as keys and dictionaries containing names of tables (keys) and
+                       their metadata as values
+        """
+        self.tables = tables
+
+    def list_namespaces(self) -> Iterable[Tuple[str]]:
+        return [*[(key,) for key in self.tables.keys()]]
+
+    def list_tables(self, namespace: str) -> Iterable[Tuple[str, str]]:
+        return [(namespace[0], table) for table in self.tables[namespace[0]].keys()]
+
+    def load_table(self, dataset_path: Tuple[str, str]) -> Table:
+        return self.tables[dataset_path[0]][dataset_path[1]]()
+
+
+class MockCatalogExceptionListingTables(MockCatalog):
+    def list_tables(self, namespace: str) -> Iterable[Tuple[str, str]]:
+        if namespace == ("no_such_namespace",):
+            raise NoSuchNamespaceError()
+        if namespace == ("generic_exception",):
+            raise Exception()
+        return super().list_tables(namespace)
+
+
+class MockCatalogExceptionListingNamespaces(MockCatalog):
+    def list_namespaces(self) -> Iterable[Tuple[str]]:
+        raise Exception()
+
+
+def test_exception_while_listing_namespaces() -> None:
+    source = with_iceberg_source(processing_threads=2)
+    mock_catalog = MockCatalogExceptionListingNamespaces({})
+    with patch(
+        "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
+    ) as get_catalog, pytest.raises(Exception):
+        get_catalog.return_value = mock_catalog
+        [*source.get_workunits_internal()]
+
+
+def test_known_exception_while_listing_tables() -> None:
+    source = with_iceberg_source(processing_threads=2)
+    mock_catalog = MockCatalogExceptionListingTables(
+        {
+            "namespaceA": {
+                "table1": lambda: Table(
+                    identifier=("namespaceA", "table1"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table1",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table1",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                )
+            },
+            "no_such_namespace": {},
+            "namespaceB": {
+                "table2": lambda: Table(
+                    identifier=("namespaceB", "table2"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceB/table2",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceB/table2",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table3": lambda: Table(
+                    identifier=("namespaceB", "table3"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceB/table3",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceB/table3",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+            },
+            "namespaceC": {
+                "table4": lambda: Table(
+                    identifier=("namespaceC", "table4"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceC/table4",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceC/table4",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                )
+            },
+            "namespaceD": {
+                "table5": lambda: Table(
+                    identifier=("namespaceD", "table5"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table5",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table5",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                )
+            },
+        }
+    )
+    with patch(
+        "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
+    ) as get_catalog:
+        get_catalog.return_value = mock_catalog
+        wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
+        assert len(wu) == 5  # ingested 5 tables, despite exception
+        urns = []
+        for unit in wu:
+            assert isinstance(unit.metadata, MetadataChangeEvent)
+            assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass)
+            urns.append(unit.metadata.proposedSnapshot.urn)
+        TestCase().assertCountEqual(
+            urns,
+            [
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)",
+            ],
+        )
+        assert source.report.warnings.total_elements == 1
+        assert source.report.failures.total_elements == 0
+        assert source.report.tables_scanned == 5
+
+
+def test_unknown_exception_while_listing_tables() -> None:
+    source = with_iceberg_source(processing_threads=2)
+    mock_catalog = MockCatalogExceptionListingTables(
+        {
+            "namespaceA": {
+                "table1": lambda: Table(
+                    identifier=("namespaceA", "table1"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table1",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table1",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                )
+            },
+            "generic_exception": {},
+            "namespaceB": {
+                "table2": lambda: Table(
+                    identifier=("namespaceB", "table2"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceB/table2",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceB/table2",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table3": lambda: Table(
+                    identifier=("namespaceB", "table3"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceB/table3",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceB/table3",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+            },
+            "namespaceC": {
+                "table4": lambda: Table(
+                    identifier=("namespaceC", "table4"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceC/table4",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceC/table4",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                )
+            },
+            "namespaceD": {
+                "table5": lambda: Table(
+                    identifier=("namespaceD", "table5"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table5",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table5",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                )
+            },
+        }
+    )
+    with patch(
+        "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
+    ) as get_catalog:
+        get_catalog.return_value = mock_catalog
+        wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
+        assert len(wu) == 5  # ingested 5 tables, despite exception
+        urns = []
+        for unit in wu:
+            assert isinstance(unit.metadata, MetadataChangeEvent)
+            assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass)
+            urns.append(unit.metadata.proposedSnapshot.urn)
+        TestCase().assertCountEqual(
+            urns,
+            [
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)",
+            ],
+        )
+        assert source.report.warnings.total_elements == 0
+        assert source.report.failures.total_elements == 1
+        assert source.report.tables_scanned == 5
+
+
+def test_proper_run_with_multiple_namespaces() -> None:
+    source = with_iceberg_source(processing_threads=3)
+    mock_catalog = MockCatalog(
+        {
+            "namespaceA": {
+                "table1": lambda: Table(
+                    identifier=("namespaceA", "table1"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table1",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table1",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                )
+            },
+            "namespaceB": {},
+        }
+    )
+    with patch(
+        "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
+    ) as get_catalog:
+        get_catalog.return_value = mock_catalog
+        wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
+        assert len(wu) == 1  # only one table processed as an MCE
+        assert isinstance(wu[0].metadata, MetadataChangeEvent)
+        assert isinstance(wu[0].metadata.proposedSnapshot, DatasetSnapshotClass)
+        snapshot: DatasetSnapshotClass = wu[0].metadata.proposedSnapshot
+        assert (
+            snapshot.urn
+            == "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)"
+        )
+
+
+def test_filtering() -> None:
+    source = with_iceberg_source(
+        processing_threads=1,
+        table_pattern=AllowDenyPattern(deny=[".*abcd.*"]),
+        namespace_pattern=AllowDenyPattern(allow=["namespace1"]),
+    )
+    mock_catalog = MockCatalog(
+        {
+            "namespace1": {
+                "table_xyz": lambda: Table(
+                    identifier=("namespace1", "table_xyz"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespace1/table_xyz",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespace1/table_xyz",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "JKLtable": lambda: Table(
+                    identifier=("namespace1", "JKLtable"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespace1/JKLtable",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespace1/JKLtable",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table_abcd": lambda: Table(
+                    identifier=("namespace1", "table_abcd"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespace1/table_abcd",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespace1/table_abcd",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "aaabcd": lambda: Table(
+                    identifier=("namespace1", "aaabcd"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespace1/aaabcd",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespace1/aaabcd",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+            },
+            "namespace2": {
+                "foo": lambda: Table(
+                    identifier=("namespace2", "foo"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespace2/foo",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespace2/foo",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "bar": lambda: Table(
+                    identifier=("namespace2", "bar"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespace2/bar",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespace2/bar",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+            },
+            "namespace3": {
+                "sales": lambda: Table(
+                    identifier=("namespace3", "sales"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespace3/sales",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespace3/sales",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "products": lambda: Table(
+                    identifier=("namespace2", "bar"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespace3/products",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespace3/products",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+            },
+        }
+    )
+    with patch(
+        "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
+    ) as get_catalog:
+        get_catalog.return_value = mock_catalog
+        wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
+        assert len(wu) == 2
+        urns = []
+        for unit in wu:
+            assert isinstance(unit.metadata, MetadataChangeEvent)
+            assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass)
+            urns.append(unit.metadata.proposedSnapshot.urn)
+        TestCase().assertCountEqual(
+            urns,
+            [
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)",
+            ],
+        )
+        assert source.report.tables_scanned == 2
+
+
+def test_handle_expected_exceptions() -> None:
+    source = with_iceberg_source(processing_threads=3)
+
+    def _raise_no_such_property_exception():
+        raise NoSuchPropertyException()
+
+    def _raise_no_such_iceberg_table_exception():
+        raise NoSuchIcebergTableError()
+
+    def _raise_file_not_found_error():
+        raise FileNotFoundError()
+
+    def _raise_no_such_table_exception():
+        raise NoSuchTableError()
+
+    def _raise_server_error():
+        raise ServerError()
+
+    mock_catalog = MockCatalog(
+        {
+            "namespaceA": {
+                "table1": lambda: Table(
+                    identifier=("namespaceA", "table1"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table1",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table1",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table2": lambda: Table(
+                    identifier=("namespaceA", "table2"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table2",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table2",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table3": lambda: Table(
+                    identifier=("namespaceA", "table3"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table3",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table3",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table4": lambda: Table(
+                    identifier=("namespaceA", "table4"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table4",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table4",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table5": _raise_no_such_property_exception,
+                "table6": _raise_no_such_table_exception,
+                "table7": _raise_file_not_found_error,
+                "table8": _raise_no_such_iceberg_table_exception,
+                "table9": _raise_server_error,
+            }
+        }
+    )
+    with patch(
+        "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
+    ) as get_catalog:
+        get_catalog.return_value = mock_catalog
+        wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
+        assert len(wu) == 4
+        urns = []
+        for unit in wu:
+            assert isinstance(unit.metadata, MetadataChangeEvent)
+            assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass)
+            urns.append(unit.metadata.proposedSnapshot.urn)
+        TestCase().assertCountEqual(
+            urns,
+            [
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)",
+            ],
+        )
+        assert source.report.warnings.total_elements == 5
+        assert source.report.failures.total_elements == 0
+        assert source.report.tables_scanned == 4
+
+
+def test_handle_unexpected_exceptions() -> None:
+    source = with_iceberg_source(processing_threads=3)
+
+    def _raise_exception():
+        raise Exception()
+
+    mock_catalog = MockCatalog(
+        {
+            "namespaceA": {
+                "table1": lambda: Table(
+                    identifier=("namespaceA", "table1"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table1",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table1",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table2": lambda: Table(
+                    identifier=("namespaceA", "table2"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table2",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table2",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table3": lambda: Table(
+                    identifier=("namespaceA", "table3"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table3",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table3",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table4": lambda: Table(
+                    identifier=("namespaceA", "table4"),
+                    metadata=TableMetadataV2(
+                        partition_specs=[PartitionSpec(spec_id=0)],
+                        location="s3://abcdefg/namespaceA/table4",
+                        last_column_id=0,
+                        schemas=[Schema(schema_id=0)],
+                    ),
+                    metadata_location="s3://abcdefg/namespaceA/table4",
+                    io=PyArrowFileIO(),
+                    catalog=None,
+                ),
+                "table5": _raise_exception,
+            }
+        }
+    )
+    with patch(
+        "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
+    ) as get_catalog:
+        get_catalog.return_value = mock_catalog
+        wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
+        assert len(wu) == 4
+        urns = []
+        for unit in wu:
+            assert isinstance(unit.metadata, MetadataChangeEvent)
+            assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass)
+            urns.append(unit.metadata.proposedSnapshot.urn)
+        TestCase().assertCountEqual(
+            urns,
+            [
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)",
+            ],
+        )
+        assert source.report.warnings.total_elements == 0
+        assert source.report.failures.total_elements == 1
+        assert source.report.tables_scanned == 4
diff --git a/metadata-ingestion/tests/unit/test_kafka_source.py b/metadata-ingestion/tests/unit/test_kafka_source.py
index b4e37d288a304..1a8afe1b956fa 100644
--- a/metadata-ingestion/tests/unit/test_kafka_source.py
+++ b/metadata-ingestion/tests/unit/test_kafka_source.py
@@ -10,6 +10,7 @@
 )
 from freezegun import freeze_time
 
+from datahub.configuration.common import ConfigurationError
 from datahub.emitter.mce_builder import (
     OwnerType,
     make_dataplatform_instance_urn,
@@ -23,7 +24,7 @@
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.kafka import KafkaSource, KafkaSourceConfig
+from datahub.ingestion.source.kafka.kafka import KafkaSource, KafkaSourceConfig
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.schema_classes import (
     BrowsePathsClass,
@@ -38,11 +39,13 @@
 
 @pytest.fixture
 def mock_admin_client():
-    with patch("datahub.ingestion.source.kafka.AdminClient", autospec=True) as mock:
+    with patch(
+        "datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True
+    ) as mock:
         yield mock
 
 
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_configuration(mock_kafka):
     ctx = PipelineContext(run_id="test")
     kafka_source = KafkaSource(
@@ -53,7 +56,7 @@ def test_kafka_source_configuration(mock_kafka):
     assert mock_kafka.call_count == 1
 
 
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_workunits_wildcard_topic(mock_kafka, mock_admin_client):
     mock_kafka_instance = mock_kafka.return_value
     mock_cluster_metadata = MagicMock()
@@ -74,7 +77,7 @@ def test_kafka_source_workunits_wildcard_topic(mock_kafka, mock_admin_client):
     assert len(workunits) == 4
 
 
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_workunits_topic_pattern(mock_kafka, mock_admin_client):
     mock_kafka_instance = mock_kafka.return_value
     mock_cluster_metadata = MagicMock()
@@ -108,7 +111,7 @@ def test_kafka_source_workunits_topic_pattern(mock_kafka, mock_admin_client):
     assert len(workunits) == 4
 
 
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_workunits_with_platform_instance(mock_kafka, mock_admin_client):
     PLATFORM_INSTANCE = "kafka_cluster"
     PLATFORM = "kafka"
@@ -160,7 +163,7 @@ def test_kafka_source_workunits_with_platform_instance(mock_kafka, mock_admin_cl
     assert f"/prod/{PLATFORM}/{PLATFORM_INSTANCE}" in browse_path_aspects[0].paths
 
 
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_workunits_no_platform_instance(mock_kafka, mock_admin_client):
     PLATFORM = "kafka"
     TOPIC_NAME = "test"
@@ -204,7 +207,7 @@ def test_kafka_source_workunits_no_platform_instance(mock_kafka, mock_admin_clie
     assert f"/prod/{PLATFORM}" in browse_path_aspects[0].paths
 
 
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_close(mock_kafka, mock_admin_client):
     mock_kafka_instance = mock_kafka.return_value
     ctx = PipelineContext(run_id="test")
@@ -223,7 +226,7 @@ def test_close(mock_kafka, mock_admin_client):
     "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient",
     autospec=True,
 )
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_workunits_schema_registry_subject_name_strategies(
     mock_kafka_consumer, mock_schema_registry_client, mock_admin_client
 ):
@@ -327,6 +330,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]:
             "topic2-key": "test.acryl.Topic2Key",
             "topic2-value": "test.acryl.Topic2Value",
         },
+        "ingest_schemas_as_entities": True,
     }
     ctx = PipelineContext(run_id="test")
     kafka_source = KafkaSource.create(source_config, ctx)
@@ -415,7 +419,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]:
     "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient",
     autospec=True,
 )
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_ignore_warnings_on_schema_type(
     mock_kafka_consumer,
     mock_schema_registry_client,
@@ -475,16 +479,15 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]:
     kafka_source = KafkaSource.create(source_config, ctx)
 
     workunits = list(kafka_source.get_workunits())
-
-    assert len(workunits) == 6
+    assert len(workunits) == 2
     if ignore_warnings_on_schema_type:
         assert not kafka_source.report.warnings
     else:
         assert kafka_source.report.warnings
 
 
-@patch("datahub.ingestion.source.kafka.AdminClient", autospec=True)
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_succeeds_with_admin_client_init_error(
     mock_kafka, mock_kafka_admin_client
 ):
@@ -513,8 +516,8 @@ def test_kafka_source_succeeds_with_admin_client_init_error(
     assert len(workunits) == 2
 
 
-@patch("datahub.ingestion.source.kafka.AdminClient", autospec=True)
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_succeeds_with_describe_configs_error(
     mock_kafka, mock_kafka_admin_client
 ):
@@ -550,7 +553,7 @@ def test_kafka_source_succeeds_with_describe_configs_error(
     "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient",
     autospec=True,
 )
-@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
+@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True)
 def test_kafka_source_topic_meta_mappings(
     mock_kafka_consumer, mock_schema_registry_client, mock_admin_client
 ):
@@ -619,6 +622,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]:
     kafka_source = KafkaSource.create(
         {
             "connection": {"bootstrap": "localhost:9092"},
+            "ingest_schemas_as_entities": True,
             "meta_mapping": {
                 "owner": {
                     "match": "^@(.*)",
@@ -736,3 +740,23 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]:
     assert workunits[7].metadata.aspectName == "glossaryTermKey"
     assert workunits[8].metadata.aspectName == "tagKey"
     assert workunits[9].metadata.aspectName == "tagKey"
+
+
+def test_kafka_source_oauth_cb_configuration():
+    with pytest.raises(
+        ConfigurationError,
+        match=(
+            "oauth_cb must be a string representing python function reference "
+            "in the format <python-module>:<function-name>."
+        ),
+    ):
+        KafkaSourceConfig.parse_obj(
+            {
+                "connection": {
+                    "bootstrap": "foobar:9092",
+                    "consumer_config": {
+                        "oauth_cb": test_kafka_ignore_warnings_on_schema_type
+                    },
+                }
+            }
+        )
diff --git a/metadata-ingestion/tests/unit/test_mlflow_source.py b/metadata-ingestion/tests/unit/test_mlflow_source.py
index ae5a42bad229d..e882296b6f331 100644
--- a/metadata-ingestion/tests/unit/test_mlflow_source.py
+++ b/metadata-ingestion/tests/unit/test_mlflow_source.py
@@ -1,6 +1,6 @@
 import datetime
 from pathlib import Path
-from typing import Any, TypeVar, Union
+from typing import Any, Union
 
 import pytest
 from mlflow import MlflowClient
@@ -11,8 +11,6 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.mlflow import MLflowConfig, MLflowSource
 
-T = TypeVar("T")
-
 
 @pytest.fixture
 def tracking_uri(tmp_path: Path) -> str:
@@ -46,7 +44,7 @@ def model_version(
     )
 
 
-def dummy_search_func(page_token: Union[None, str], **kwargs: Any) -> PagedList[T]:
+def dummy_search_func(page_token: Union[None, str], **kwargs: Any) -> PagedList[str]:
     dummy_pages = dict(
         page_1=PagedList(items=["a", "b"], token="page_2"),
         page_2=PagedList(items=["c", "d"], token="page_3"),
@@ -138,3 +136,16 @@ def test_make_external_link_remote(source, model_version):
     url = source._make_external_url(model_version)
 
     assert url == expected_url
+
+
+def test_make_external_link_remote_via_config(source, model_version):
+    custom_base_url = "https://custom-server.org"
+    source.config.base_external_url = custom_base_url
+    source.client = MlflowClient(
+        tracking_uri="https://dummy-mlflow-tracking-server.org"
+    )
+    expected_url = f"{custom_base_url}/#/models/{model_version.name}/versions/{model_version.version}"
+
+    url = source._make_external_url(model_version)
+
+    assert url == expected_url
diff --git a/metadata-ingestion/tests/unit/test_neo4j_source.py b/metadata-ingestion/tests/unit/test_neo4j_source.py
new file mode 100644
index 0000000000000..62586718e8606
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_neo4j_source.py
@@ -0,0 +1,221 @@
+import unittest
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.source.neo4j.neo4j_source import Neo4jConfig, Neo4jSource
+
+
+@pytest.fixture
+def tracking_uri(tmp_path: Path) -> str:
+    # return str(tmp_path / "neo4j")
+    return "neo4j+ssc://host:7687"
+
+
+@pytest.fixture
+def source(tracking_uri: str) -> Neo4jSource:
+    return Neo4jSource(
+        ctx=PipelineContext(run_id="neo4j-test"),
+        config=Neo4jConfig(
+            uri=tracking_uri, env="Prod", username="test", password="test"
+        ),
+    )
+
+
+def data():
+    return [
+        {
+            "key": "Node_1",
+            "value": {
+                "count": 433026,
+                "relationships": {
+                    "RELATIONSHIP_1": {
+                        "count": 1,
+                        "properties": {
+                            "Relationship1_Property1": {
+                                "existence": False,
+                                "type": "STRING",
+                                "indexed": False,
+                                "array": False,
+                            }
+                        },
+                        "direction": "in",
+                        "labels": ["Node_2"],
+                    }
+                },
+                "RELATIONSHIP_2": {
+                    "count": 2,
+                    "properties": {
+                        "Relationship2_Property1": {
+                            "existence": False,
+                            "type": "STRING",
+                            "indexed": False,
+                            "array": False,
+                        }
+                    },
+                    "direction": "in",
+                    "labels": ["Node_3"],
+                },
+                "type": "node",
+                "properties": {
+                    "Node1_Property1": {
+                        "existence": False,
+                        "type": "DATE",
+                        "indexed": False,
+                        "unique": False,
+                    },
+                    "Node1_Property2": {
+                        "existence": False,
+                        "type": "STRING",
+                        "indexed": False,
+                        "unique": False,
+                    },
+                    "Node1_Property3": {
+                        "existence": False,
+                        "type": "STRING",
+                        "indexed": False,
+                        "unique": False,
+                    },
+                },
+                "labels": [],
+            },
+        },
+        {
+            "key": "Node_2",
+            "value": {
+                "count": 3,
+                "relationships": {
+                    "RELATIONSHIP_1": {
+                        "count": 1,
+                        "properties": {
+                            "Relationship1_Property1": {
+                                "existence": False,
+                                "type": "STRING",
+                                "indexed": False,
+                                "array": False,
+                            }
+                        },
+                        "direction": "out",
+                        "labels": ["Node_2"],
+                    }
+                },
+                "type": "node",
+                "properties": {
+                    "Node2_Property1": {
+                        "existence": False,
+                        "type": "DATE",
+                        "indexed": False,
+                        "unique": False,
+                    },
+                    "Node2_Property2": {
+                        "existence": False,
+                        "type": "STRING",
+                        "indexed": False,
+                        "unique": False,
+                    },
+                    "Node2_Property3": {
+                        "existence": False,
+                        "type": "STRING",
+                        "indexed": False,
+                        "unique": False,
+                    },
+                },
+                "labels": [],
+            },
+        },
+        {
+            "key": "RELATIONSHIP_1",
+            "value": {
+                "count": 4,
+                "type": "relationship",
+                "properties": {
+                    "Relationship1_Property1": {
+                        "existence": False,
+                        "type": "STRING",
+                        "indexed": False,
+                        "array": False,
+                    }
+                },
+            },
+        },
+    ]
+
+
+def test_process_nodes(source):
+    df = source.process_nodes(data=data())
+    assert type(df) is pd.DataFrame
+
+
+def test_process_relationships(source):
+    df = source.process_relationships(
+        data=data(), node_df=source.process_nodes(data=data())
+    )
+    assert type(df) is pd.DataFrame
+
+
+def test_get_obj_type(source):
+    results = data()
+    assert source.get_obj_type(results[0]["value"]) == "node"
+    assert source.get_obj_type(results[1]["value"]) == "node"
+    assert source.get_obj_type(results[2]["value"]) == "relationship"
+
+
+def test_get_node_description(source):
+    results = data()
+    df = source.process_nodes(data=data())
+    assert (
+        source.get_node_description(results[0], df)
+        == "(Node_1)<-[RELATIONSHIP_1]-(Node_2)"
+    )
+    assert (
+        source.get_node_description(results[1], df)
+        == "(Node_2)-[RELATIONSHIP_1]->(Node_2)"
+    )
+
+
+def test_get_property_data_types(source):
+    results = data()
+    assert source.get_property_data_types(results[0]["value"]["properties"]) == [
+        {"Node1_Property1": "DATE"},
+        {"Node1_Property2": "STRING"},
+        {"Node1_Property3": "STRING"},
+    ]
+    assert source.get_property_data_types(results[1]["value"]["properties"]) == [
+        {"Node2_Property1": "DATE"},
+        {"Node2_Property2": "STRING"},
+        {"Node2_Property3": "STRING"},
+    ]
+    assert source.get_property_data_types(results[2]["value"]["properties"]) == [
+        {"Relationship1_Property1": "STRING"}
+    ]
+
+
+def test_get_properties(source):
+    results = data()
+    assert list(source.get_properties(results[0]["value"]).keys()) == [
+        "Node1_Property1",
+        "Node1_Property2",
+        "Node1_Property3",
+    ]
+    assert list(source.get_properties(results[1]["value"]).keys()) == [
+        "Node2_Property1",
+        "Node2_Property2",
+        "Node2_Property3",
+    ]
+    assert list(source.get_properties(results[2]["value"]).keys()) == [
+        "Relationship1_Property1"
+    ]
+
+
+def test_get_relationships(source):
+    results = data()
+    record = list(
+        results[0]["value"]["relationships"].keys()
+    )  # Get the first key from the dict_keys
+    assert record == ["RELATIONSHIP_1"]
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/metadata-ingestion/tests/unit/test_oracle_source.py b/metadata-ingestion/tests/unit/test_oracle_source.py
index 18d4e85296e36..0477044354576 100644
--- a/metadata-ingestion/tests/unit/test_oracle_source.py
+++ b/metadata-ingestion/tests/unit/test_oracle_source.py
@@ -21,7 +21,7 @@ def test_oracle_config():
     )
     assert (
         config.get_sql_alchemy_url()
-        == "oracle+cx_oracle://user:password@host:1521/?service_name=svc01"
+        == "oracle://user:password@host:1521/?service_name=svc01"
     )
 
     with pytest.raises(ValueError):
diff --git a/metadata-ingestion/tests/unit/test_powerbi_parser.py b/metadata-ingestion/tests/unit/test_powerbi_parser.py
index e53e8d7aee16f..a487a3a5b87f8 100644
--- a/metadata-ingestion/tests/unit/test_powerbi_parser.py
+++ b/metadata-ingestion/tests/unit/test_powerbi_parser.py
@@ -1,13 +1,15 @@
 import pytest
 
 from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.source.powerbi.config import PowerBiDashboardSourceConfig
+from datahub.ingestion.source.powerbi.config import (
+    PowerBiDashboardSourceConfig,
+    PowerBiDashboardSourceReport,
+)
 from datahub.ingestion.source.powerbi.dataplatform_instance_resolver import (
     ResolvePlatformInstanceFromDatasetTypeMapping,
 )
-from datahub.ingestion.source.powerbi.m_query.resolver import (
-    MSSqlDataPlatformTableCreator,
-)
+from datahub.ingestion.source.powerbi.m_query.pattern_handler import MSSqlLineage
+from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table
 
 
 @pytest.fixture
@@ -17,8 +19,16 @@ def creator():
         client_id="test-client-id",
         client_secret="test-client-secret",
     )
-    return MSSqlDataPlatformTableCreator(
+
+    table = Table(
+        name="test_table",
+        full_name="db.schema.test_table",
+    )
+
+    return MSSqlLineage(
         ctx=PipelineContext(run_id="test-run-id"),
+        table=table,
+        reporter=PowerBiDashboardSourceReport(),
         config=config,
         platform_instance_resolver=ResolvePlatformInstanceFromDatasetTypeMapping(
             config
diff --git a/metadata-ingestion/tests/unit/test_preset_source.py b/metadata-ingestion/tests/unit/test_preset_source.py
index d97db651f4c79..dc81f4c8284d5 100644
--- a/metadata-ingestion/tests/unit/test_preset_source.py
+++ b/metadata-ingestion/tests/unit/test_preset_source.py
@@ -20,3 +20,23 @@ def test_set_display_uri():
     assert config.connect_uri == ""
     assert config.manager_uri == "https://api.app.preset.io"
     assert config.display_uri == display_uri
+
+
+def test_preset_config_parsing():
+    preset_config = {
+        "connect_uri": "https://preset.io",
+        "api_key": "dummy_api_key",
+        "api_secret": "dummy_api_secret",
+        "manager_uri": "https://api.app.preset.io",
+    }
+
+    # Tests if SupersetConfig fields are parsed extra fields correctly
+    config = PresetConfig.parse_obj(preset_config)
+
+    # Test Preset-specific fields
+    assert config.api_key == "dummy_api_key"
+    assert config.api_secret == "dummy_api_secret"
+    assert config.manager_uri == "https://api.app.preset.io"
+
+    # Test that regular Superset fields are still parsed
+    assert config.connect_uri == "https://preset.io"
diff --git a/metadata-ingestion/tests/unit/test_redash_source.py b/metadata-ingestion/tests/unit/test_redash_source.py
index 2982fe76c4d4e..32ab200847dc6 100644
--- a/metadata-ingestion/tests/unit/test_redash_source.py
+++ b/metadata-ingestion/tests/unit/test_redash_source.py
@@ -710,9 +710,9 @@ def test_get_chart_snapshot_parse_table_names_from_sql(mocked_data_source):
                 ),
                 chartUrl="http://localhost:5000/queries/4#10",
                 inputs=[
-                    "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.order_items,PROD)",
-                    "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.orders,PROD)",
-                    "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.staffs,PROD)",
+                    "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.order_items,PROD)",
+                    "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.orders,PROD)",
+                    "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.staffs,PROD)",
                 ],
                 type="PIE",
             )
diff --git a/metadata-ingestion/tests/unit/test_sql_common.py b/metadata-ingestion/tests/unit/test_sql_common.py
index a98bf64171122..cfb8f55bd977f 100644
--- a/metadata-ingestion/tests/unit/test_sql_common.py
+++ b/metadata-ingestion/tests/unit/test_sql_common.py
@@ -38,7 +38,7 @@ def test_generate_foreign_key():
         "referred_columns": ["test_referred_column"],  # type: ignore
     }
     foreign_key = source.get_foreign_key_metadata(
-        dataset_urn="test_urn",
+        dataset_urn="urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.base_urn,PROD)",
         schema="test_schema",
         fk_dict=fk_dict,
         inspector=mock.Mock(),
@@ -48,7 +48,9 @@ def test_generate_foreign_key():
     assert [
         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:TEST,test_referred_schema.test_table,PROD),test_referred_column)"
     ] == foreign_key.foreignFields
-    assert ["urn:li:schemaField:(test_urn,test_column)"] == foreign_key.sourceFields
+    assert [
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.base_urn,PROD),test_column)"
+    ] == foreign_key.sourceFields
 
 
 def test_use_source_schema_for_foreign_key_if_not_specified():
@@ -60,7 +62,7 @@ def test_use_source_schema_for_foreign_key_if_not_specified():
         "referred_columns": ["test_referred_column"],  # type: ignore
     }
     foreign_key = source.get_foreign_key_metadata(
-        dataset_urn="test_urn",
+        dataset_urn="urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.base_urn,PROD)",
         schema="test_schema",
         fk_dict=fk_dict,
         inspector=mock.Mock(),
@@ -70,7 +72,9 @@ def test_use_source_schema_for_foreign_key_if_not_specified():
     assert [
         "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.test_table,PROD),test_referred_column)"
     ] == foreign_key.foreignFields
-    assert ["urn:li:schemaField:(test_urn,test_column)"] == foreign_key.sourceFields
+    assert [
+        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.base_urn,PROD),test_column)"
+    ] == foreign_key.sourceFields
 
 
 PLATFORM_FROM_SQLALCHEMY_URI_TEST_CASES: Dict[str, str] = {
diff --git a/metadata-ingestion/tests/unit/test_sql_types.py b/metadata-ingestion/tests/unit/test_sql_types.py
new file mode 100644
index 0000000000000..ebe5ade115cdd
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_sql_types.py
@@ -0,0 +1,78 @@
+import pytest
+
+from datahub.ingestion.source.sql.sql_types import (
+    ATHENA_SQL_TYPES_MAP,
+    TRINO_SQL_TYPES_MAP,
+    resolve_athena_modified_type,
+    resolve_sql_type,
+    resolve_trino_modified_type,
+)
+from datahub.metadata.schema_classes import BooleanTypeClass, StringTypeClass
+
+
+@pytest.mark.parametrize(
+    "data_type, expected_data_type",
+    [
+        ("boolean", "boolean"),
+        ("tinyint", "tinyint"),
+        ("smallint", "smallint"),
+        ("int", "int"),
+        ("integer", "integer"),
+        ("bigint", "bigint"),
+        ("real", "real"),
+        ("double", "double"),
+        ("decimal(10,0)", "decimal"),
+        ("varchar(20)", "varchar"),
+        ("char", "char"),
+        ("varbinary", "varbinary"),
+        ("json", "json"),
+        ("date", "date"),
+        ("time", "time"),
+        ("time(12)", "time"),
+        ("timestamp", "timestamp"),
+        ("timestamp(3)", "timestamp"),
+        ("row(x bigint, y double)", "row"),
+        ("array(row(x bigint, y double))", "array"),
+        ("map(varchar, varchar)", "map"),
+    ],
+)
+def test_resolve_trino_modified_type(data_type, expected_data_type):
+    assert (
+        resolve_trino_modified_type(data_type)
+        == TRINO_SQL_TYPES_MAP[expected_data_type]
+    )
+
+
+@pytest.mark.parametrize(
+    "data_type, expected_data_type",
+    [
+        ("boolean", "boolean"),
+        ("tinyint", "tinyint"),
+        ("smallint", "smallint"),
+        ("int", "int"),
+        ("integer", "integer"),
+        ("bigint", "bigint"),
+        ("float", "float"),
+        ("double", "double"),
+        ("decimal(10,0)", "decimal"),
+        ("varchar(20)", "varchar"),
+        ("char", "char"),
+        ("binary", "binary"),
+        ("date", "date"),
+        ("timestamp", "timestamp"),
+        ("timestamp(3)", "timestamp"),
+        ("struct<x timestamp(3), y timestamp>", "struct"),
+        ("array<struct<x bigint, y double>>", "array"),
+        ("map<varchar, varchar>", "map"),
+    ],
+)
+def test_resolve_athena_modified_type(data_type, expected_data_type):
+    assert (
+        resolve_athena_modified_type(data_type)
+        == ATHENA_SQL_TYPES_MAP[expected_data_type]
+    )
+
+
+def test_resolve_sql_type() -> None:
+    assert resolve_sql_type("boolean") == BooleanTypeClass()
+    assert resolve_sql_type("varchar") == StringTypeClass()
diff --git a/metadata-ingestion/tests/unit/test_tableau_source.py b/metadata-ingestion/tests/unit/test_tableau_source.py
index c81aa0bd8a1b1..44e59decaecbd 100644
--- a/metadata-ingestion/tests/unit/test_tableau_source.py
+++ b/metadata-ingestion/tests/unit/test_tableau_source.py
@@ -182,8 +182,14 @@ def test_get_filter_pages_simple():
     assert get_filter_pages(filter_dict, 10) == [filter_dict]
 
 
-def test_get_filter_pages_non_id_large_filter_passthrough():
-    projects = [f"project{i}" for i in range(20000)]
+def test_get_filter_pages_non_id_large_filter():
+    projects = [f"project{i}" for i in range(10)]
+    filter_dict = {c.PROJECT_NAME_WITH_IN: projects}
+    assert get_filter_pages(filter_dict, 10) == [filter_dict]
+
+
+def test_get_filter_pages_for_single_key():
+    projects = ["project1"]
     filter_dict = {c.PROJECT_NAME_WITH_IN: projects}
     assert get_filter_pages(filter_dict, 10) == [filter_dict]
 
diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py
index 46c6390b184d3..5151be9c8b199 100644
--- a/metadata-ingestion/tests/unit/test_transform_dataset.py
+++ b/metadata-ingestion/tests/unit/test_transform_dataset.py
@@ -1,17 +1,7 @@
 import json
 import re
 from datetime import datetime, timezone
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    List,
-    MutableSequence,
-    Optional,
-    Type,
-    Union,
-    cast,
-)
+from typing import Any, Dict, List, MutableSequence, Optional, Type, Union, cast
 from unittest import mock
 from uuid import uuid4
 
@@ -24,7 +14,7 @@
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api import workunit
 from datahub.ingestion.api.common import EndOfStream, PipelineContext, RecordEnvelope
-from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
+from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.ingestion.transformer.add_dataset_browse_path import (
     AddDatasetBrowsePathTransformer,
@@ -66,6 +56,7 @@
 from datahub.ingestion.transformer.dataset_domain import (
     PatternAddDatasetDomain,
     SimpleAddDatasetDomain,
+    TransformerOnConflict,
 )
 from datahub.ingestion.transformer.dataset_domain_based_on_tags import (
     DatasetTagDomainMapper,
@@ -131,7 +122,7 @@ def make_generic_dataset_mcp(
 ) -> MetadataChangeProposalWrapper:
     return MetadataChangeProposalWrapper(
         entityUrn=entity_urn,
-        entityType=Urn.create_from_string(entity_urn).get_type(),
+        entityType=Urn.from_string(entity_urn).get_type(),
         aspectName=aspect_name,
         changeType="UPSERT",
         aspect=aspect,
@@ -147,7 +138,7 @@ def make_generic_container_mcp(
         aspect = models.StatusClass(removed=False)
     return MetadataChangeProposalWrapper(
         entityUrn=entity_urn,
-        entityType=Urn.create_from_string(entity_urn).get_type(),
+        entityType=Urn.from_string(entity_urn).get_type(),
         aspectName=aspect_name,
         changeType="UPSERT",
         aspect=aspect,
@@ -229,7 +220,7 @@ def make_dataset_with_properties() -> models.MetadataChangeEventClass:
     )
 
 
-def test_simple_dataset_ownership_transformation(mock_time):
+def test_dataset_ownership_transformation(mock_time):
     no_owner_aspect = make_generic_dataset()
 
     with_owner_aspect = make_dataset_with_owner()
@@ -263,7 +254,7 @@ def test_simple_dataset_ownership_transformation(mock_time):
         transformer.transform([RecordEnvelope(input, metadata={}) for input in inputs])
     )
 
-    assert len(outputs) == len(inputs) + 1
+    assert len(outputs) == len(inputs) + 2
 
     # Check the first entry.
     first_ownership_aspect = builder.get_aspect_if_available(
@@ -296,11 +287,21 @@ def test_simple_dataset_ownership_transformation(mock_time):
         ]
     )
 
+    third_ownership_aspect = outputs[4].record.aspect
+    assert third_ownership_aspect
+    assert len(third_ownership_aspect.owners) == 2
+    assert all(
+        [
+            owner.type == models.OwnershipTypeClass.DATAOWNER and owner.typeUrn is None
+            for owner in second_ownership_aspect.owners
+        ]
+    )
+
     # Verify that the third entry is unchanged.
     assert inputs[2] == outputs[2].record
 
     # Verify that the last entry is EndOfStream
-    assert inputs[3] == outputs[4].record
+    assert inputs[-1] == outputs[-1].record
 
 
 def test_simple_dataset_ownership_with_type_transformation(mock_time):
@@ -1012,6 +1013,7 @@ def test_pattern_dataset_ownership_transformation(mock_time):
                 "rules": {
                     ".*example1.*": [builder.make_user_urn("person1")],
                     ".*example2.*": [builder.make_user_urn("person2")],
+                    ".*dag_abc.*": [builder.make_user_urn("person2")],
                 }
             },
             "ownership_type": "DATAOWNER",
@@ -1023,7 +1025,9 @@ def test_pattern_dataset_ownership_transformation(mock_time):
         transformer.transform([RecordEnvelope(input, metadata={}) for input in inputs])
     )
 
-    assert len(outputs) == len(inputs) + 1  # additional MCP due to the no-owner MCE
+    assert (
+        len(outputs) == len(inputs) + 2
+    )  # additional MCP due to the no-owner MCE + datajob
 
     # Check the first entry.
     assert inputs[0] == outputs[0].record
@@ -1051,6 +1055,16 @@ def test_pattern_dataset_ownership_transformation(mock_time):
         ]
     )
 
+    third_ownership_aspect = outputs[4].record.aspect
+    assert third_ownership_aspect
+    assert len(third_ownership_aspect.owners) == 1
+    assert all(
+        [
+            owner.type == models.OwnershipTypeClass.DATAOWNER
+            for owner in third_ownership_aspect.owners
+        ]
+    )
+
     # Verify that the third entry is unchanged.
     assert inputs[2] == outputs[2].record
 
@@ -1106,7 +1120,7 @@ def test_pattern_dataset_ownership_with_invalid_type_transformation(mock_time):
 
 
 def test_pattern_container_and_dataset_ownership_transformation(
-    mock_time, mock_datahub_graph
+    mock_time, mock_datahub_graph_instance
 ):
     def fake_get_aspect(
         entity_urn: str,
@@ -1127,18 +1141,18 @@ def fake_get_aspect(
     pipeline_context = PipelineContext(
         run_id="test_pattern_container_and_dataset_ownership_transformation"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_aspect = fake_get_aspect  # type: ignore
 
     # No owner aspect for the first dataset
-    no_owner_aspect = models.MetadataChangeEventClass(
+    no_owner_aspect_dataset = models.MetadataChangeEventClass(
         proposedSnapshot=models.DatasetSnapshotClass(
             urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,example1,PROD)",
             aspects=[models.StatusClass(removed=False)],
         ),
     )
     # Dataset with an existing owner
-    with_owner_aspect = models.MetadataChangeEventClass(
+    with_owner_aspect_dataset = models.MetadataChangeEventClass(
         proposedSnapshot=models.DatasetSnapshotClass(
             urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,example2,PROD)",
             aspects=[
@@ -1157,8 +1171,7 @@ def fake_get_aspect(
         ),
     )
 
-    # Not a dataset, should be ignored
-    not_a_dataset = models.MetadataChangeEventClass(
+    datajob = models.MetadataChangeEventClass(
         proposedSnapshot=models.DataJobSnapshotClass(
             urn="urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)",
             aspects=[
@@ -1172,9 +1185,9 @@ def fake_get_aspect(
     )
 
     inputs = [
-        no_owner_aspect,
-        with_owner_aspect,
-        not_a_dataset,
+        no_owner_aspect_dataset,
+        with_owner_aspect_dataset,
+        datajob,
         EndOfStream(),
     ]
 
@@ -1185,6 +1198,7 @@ def fake_get_aspect(
                 "rules": {
                     ".*example1.*": [builder.make_user_urn("person1")],
                     ".*example2.*": [builder.make_user_urn("person2")],
+                    ".*dag_abc.*": [builder.make_user_urn("person3")],
                 }
             },
             "ownership_type": "DATAOWNER",
@@ -1197,9 +1211,9 @@ def fake_get_aspect(
         transformer.transform([RecordEnvelope(input, metadata={}) for input in inputs])
     )
 
-    assert len(outputs) == len(inputs) + 3
+    assert len(outputs) == len(inputs) + 4
 
-    # Check the first entry.
+    # Check that DatasetSnapshotClass has not changed
     assert inputs[0] == outputs[0].record
 
     # Check the ownership for the first dataset (example1)
@@ -1226,12 +1240,16 @@ def fake_get_aspect(
         ]
     )
 
+    third_ownership_aspect = outputs[4].record.aspect
+    assert third_ownership_aspect
+    assert len(third_ownership_aspect.owners) == 1  # new for datajob
+
     # Check container ownerships
     for i in range(2):
-        container_ownership_aspect = outputs[i + 4].record.aspect
+        container_ownership_aspect = outputs[i + 5].record.aspect
         assert container_ownership_aspect
         ownership = json.loads(container_ownership_aspect.value.decode("utf-8"))
-        assert len(ownership) == 2
+        assert len(ownership) == 3
         assert ownership[0]["value"]["owner"] == builder.make_user_urn("person1")
         assert ownership[1]["value"]["owner"] == builder.make_user_urn("person2")
 
@@ -1240,7 +1258,7 @@ def fake_get_aspect(
 
 
 def test_pattern_container_and_dataset_ownership_with_no_container(
-    mock_time, mock_datahub_graph
+    mock_time, mock_datahub_graph_instance
 ):
     def fake_get_aspect(
         entity_urn: str,
@@ -1252,7 +1270,7 @@ def fake_get_aspect(
     pipeline_context = PipelineContext(
         run_id="test_pattern_container_and_dataset_ownership_with_no_container"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_aspect = fake_get_aspect  # type: ignore
 
     # No owner aspect for the first dataset
@@ -1357,7 +1375,7 @@ def fake_get_aspect(
 
 
 def test_pattern_container_and_dataset_ownership_with_no_match(
-    mock_time, mock_datahub_graph
+    mock_time, mock_datahub_graph_instance
 ):
     def fake_get_aspect(
         entity_urn: str,
@@ -1375,7 +1393,7 @@ def fake_get_aspect(
     pipeline_context = PipelineContext(
         run_id="test_pattern_container_and_dataset_ownership_with_no_match"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_aspect = fake_get_aspect  # type: ignore
 
     # No owner aspect for the first dataset
@@ -1598,10 +1616,10 @@ def run_simple_add_dataset_properties_transformer_semantics(
     semantics: TransformerSemantics,
     new_properties: dict,
     server_properties: dict,
-    mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph],
+    mock_datahub_graph_instance: DataHubGraph,
 ) -> List[RecordEnvelope]:
     pipeline_context = PipelineContext(run_id="test_pattern_dataset_schema_terms")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # fake the server response
     def fake_dataset_properties(entity_urn: str) -> models.DatasetPropertiesClass:
@@ -1624,7 +1642,7 @@ def fake_dataset_properties(entity_urn: str) -> models.DatasetPropertiesClass:
     return output
 
 
-def test_simple_add_dataset_properties_overwrite(mock_datahub_graph):
+def test_simple_add_dataset_properties_overwrite(mock_datahub_graph_instance):
     new_properties = {"new-simple-property": "new-value"}
     server_properties = {"p1": "value1"}
 
@@ -1632,7 +1650,7 @@ def test_simple_add_dataset_properties_overwrite(mock_datahub_graph):
         semantics=TransformerSemantics.OVERWRITE,
         new_properties=new_properties,
         server_properties=server_properties,
-        mock_datahub_graph=mock_datahub_graph,
+        mock_datahub_graph_instance=mock_datahub_graph_instance,
     )
 
     assert len(output) == 2
@@ -1648,7 +1666,7 @@ def test_simple_add_dataset_properties_overwrite(mock_datahub_graph):
     }
 
 
-def test_simple_add_dataset_properties_patch(mock_datahub_graph):
+def test_simple_add_dataset_properties_patch(mock_datahub_graph_instance):
     new_properties = {"new-simple-property": "new-value"}
     server_properties = {"p1": "value1"}
 
@@ -1656,7 +1674,7 @@ def test_simple_add_dataset_properties_patch(mock_datahub_graph):
         semantics=TransformerSemantics.PATCH,
         new_properties=new_properties,
         server_properties=server_properties,
-        mock_datahub_graph=mock_datahub_graph,
+        mock_datahub_graph_instance=mock_datahub_graph_instance,
     )
 
     assert len(output) == 2
@@ -2334,24 +2352,24 @@ def run_container_transformer_pipeline(
     return outputs
 
 
-def test_simple_add_dataset_domain_aspect_name(mock_datahub_graph):
+def test_simple_add_dataset_domain_aspect_name(mock_datahub_graph_instance):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     transformer = SimpleAddDatasetDomain.create({"domains": []}, pipeline_context)
     assert transformer.aspect_name() == models.DomainsClass.ASPECT_NAME
 
 
-def test_simple_add_dataset_domain(mock_datahub_graph):
+def test_simple_add_dataset_domain(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
 
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=SimpleAddDatasetDomain,
@@ -2372,14 +2390,14 @@ def test_simple_add_dataset_domain(mock_datahub_graph):
     assert acryl_domain in transformed_aspect.domains
 
 
-def test_simple_add_dataset_domain_mce_support(mock_datahub_graph):
+def test_simple_add_dataset_domain_mce_support(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
 
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=SimpleAddDatasetDomain,
@@ -2403,14 +2421,14 @@ def test_simple_add_dataset_domain_mce_support(mock_datahub_graph):
     assert acryl_domain in transformed_aspect.domains
 
 
-def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph):
+def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
 
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=SimpleAddDatasetDomain,
@@ -2431,13 +2449,13 @@ def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph):
     assert acryl_domain in transformed_aspect.domains
 
 
-def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph):
+def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     server_domain = builder.make_domain_urn("test.io")
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_domain(entity_urn: str) -> models.DomainsClass:
@@ -2469,14 +2487,14 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass:
 
 
 def test_simple_add_dataset_domain_semantics_patch(
-    pytestconfig, tmp_path, mock_time, mock_datahub_graph
+    pytestconfig, tmp_path, mock_time, mock_datahub_graph_instance
 ):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     server_domain = builder.make_domain_urn("test.io")
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_domain(entity_urn: str) -> models.DomainsClass:
@@ -2508,11 +2526,86 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass:
     assert server_domain in transformed_aspect.domains
 
 
-def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph):
+def test_simple_add_dataset_domain_on_conflict_do_nothing(
+    pytestconfig, tmp_path, mock_time, mock_datahub_graph_instance
+):
+    acryl_domain = builder.make_domain_urn("acryl.io")
+    datahub_domain = builder.make_domain_urn("datahubproject.io")
+    server_domain = builder.make_domain_urn("test.io")
+
+    pipeline_context = PipelineContext(run_id="transformer_pipe_line")
+    pipeline_context.graph = mock_datahub_graph_instance
+
+    # Return fake aspect to simulate server behaviour
+    def fake_get_domain(entity_urn: str) -> models.DomainsClass:
+        return models.DomainsClass(domains=[server_domain])
+
+    pipeline_context.graph.get_domain = fake_get_domain  # type: ignore
+
+    output = run_dataset_transformer_pipeline(
+        transformer_type=SimpleAddDatasetDomain,
+        aspect=models.DomainsClass(domains=[datahub_domain]),
+        config={
+            "replace_existing": False,
+            "semantics": TransformerSemantics.PATCH,
+            "domains": [acryl_domain],
+            "on_conflict": TransformerOnConflict.DO_NOTHING,
+        },
+        pipeline_context=pipeline_context,
+    )
+
+    assert len(output) == 1
+    assert output[0] is not None
+    assert output[0].record is not None
+    assert isinstance(output[0].record, EndOfStream)
+
+
+def test_simple_add_dataset_domain_on_conflict_do_nothing_no_conflict(
+    pytestconfig, tmp_path, mock_time, mock_datahub_graph_instance
+):
+    acryl_domain = builder.make_domain_urn("acryl.io")
+    datahub_domain = builder.make_domain_urn("datahubproject.io")
+    irrelevant_domain = builder.make_domain_urn("test.io")
+
+    pipeline_context = PipelineContext(run_id="transformer_pipe_line")
+    pipeline_context.graph = mock_datahub_graph_instance
+
+    # Return fake aspect to simulate server behaviour
+    def fake_get_domain(entity_urn: str) -> models.DomainsClass:
+        return models.DomainsClass(domains=[])
+
+    pipeline_context.graph.get_domain = fake_get_domain  # type: ignore
+
+    output = run_dataset_transformer_pipeline(
+        transformer_type=SimpleAddDatasetDomain,
+        aspect=models.DomainsClass(domains=[datahub_domain]),
+        config={
+            "replace_existing": False,
+            "semantics": TransformerSemantics.PATCH,
+            "domains": [acryl_domain],
+            "on_conflict": TransformerOnConflict.DO_NOTHING,
+        },
+        pipeline_context=pipeline_context,
+    )
+
+    assert len(output) == 2
+    assert output[0] is not None
+    assert output[0].record is not None
+    assert isinstance(output[0].record, MetadataChangeProposalWrapper)
+    assert output[0].record.aspect is not None
+    assert isinstance(output[0].record.aspect, models.DomainsClass)
+    transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
+    assert len(transformed_aspect.domains) == 2
+    assert datahub_domain in transformed_aspect.domains
+    assert acryl_domain in transformed_aspect.domains
+    assert irrelevant_domain not in transformed_aspect.domains
+
+
+def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph_instance):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     transformer = PatternAddDatasetDomain.create(
         {"domain_pattern": {"rules": {}}}, pipeline_context
@@ -2520,7 +2613,7 @@ def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph):
     assert transformer.aspect_name() == models.DomainsClass.ASPECT_NAME
 
 
-def test_pattern_add_dataset_domain_match(mock_datahub_graph):
+def test_pattern_add_dataset_domain_match(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*"
@@ -2528,7 +2621,7 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=PatternAddDatasetDomain,
@@ -2551,7 +2644,7 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph):
     assert acryl_domain in transformed_aspect.domains
 
 
-def test_pattern_add_dataset_domain_no_match(mock_datahub_graph):
+def test_pattern_add_dataset_domain_no_match(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*"
@@ -2559,7 +2652,7 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=PatternAddDatasetDomain,
@@ -2582,7 +2675,7 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph):
     assert acryl_domain not in transformed_aspect.domains
 
 
-def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph):
+def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*"
@@ -2590,7 +2683,7 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=PatternAddDatasetDomain,
@@ -2614,7 +2707,9 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph):
     assert acryl_domain in transformed_aspect.domains
 
 
-def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph):
+def test_pattern_add_dataset_domain_replace_existing_no_match(
+    mock_datahub_graph_instance,
+):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*"
@@ -2622,7 +2717,7 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=PatternAddDatasetDomain,
@@ -2644,14 +2739,14 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph
     assert len(transformed_aspect.domains) == 0
 
 
-def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph):
+def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     server_domain = builder.make_domain_urn("test.io")
     pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*"
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_domain(entity_urn: str) -> models.DomainsClass:
@@ -2683,7 +2778,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass:
 
 
 def test_pattern_add_dataset_domain_semantics_patch(
-    pytestconfig, tmp_path, mock_time, mock_datahub_graph
+    pytestconfig, tmp_path, mock_time, mock_datahub_graph_instance
 ):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
@@ -2691,7 +2786,7 @@ def test_pattern_add_dataset_domain_semantics_patch(
     pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*"
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_domain(entity_urn: str) -> models.DomainsClass:
@@ -2723,9 +2818,11 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass:
     assert server_domain in transformed_aspect.domains
 
 
-def test_simple_dataset_ownership_transformer_semantics_patch(mock_datahub_graph):
+def test_simple_dataset_ownership_transformer_semantics_patch(
+    mock_datahub_graph_instance,
+):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     server_owner: str = builder.make_owner_urn(
         "mohd@acryl.io", owner_type=builder.OwnerType.USER
@@ -2783,7 +2880,9 @@ def fake_ownership_class(entity_urn: str) -> models.OwnershipClass:
     assert server_owner in owner_urns
 
 
-def test_pattern_container_and_dataset_domain_transformation(mock_datahub_graph):
+def test_pattern_container_and_dataset_domain_transformation(
+    mock_datahub_graph_instance,
+):
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     acryl_domain = builder.make_domain_urn("acryl_domain")
     server_domain = builder.make_domain_urn("server_domain")
@@ -2807,7 +2906,7 @@ def fake_get_aspect(
     pipeline_context = PipelineContext(
         run_id="test_pattern_container_and_dataset_domain_transformation"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_aspect = fake_get_aspect  # type: ignore
 
     with_domain_aspect = make_generic_dataset_mcp(
@@ -2889,7 +2988,7 @@ def fake_get_aspect(
 
 
 def test_pattern_container_and_dataset_domain_transformation_with_no_container(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     acryl_domain = builder.make_domain_urn("acryl_domain")
@@ -2905,7 +3004,7 @@ def fake_get_aspect(
     pipeline_context = PipelineContext(
         run_id="test_pattern_container_and_dataset_domain_transformation_with_no_container"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_aspect = fake_get_aspect  # type: ignore
 
     with_domain_aspect = make_generic_dataset_mcp(
@@ -2955,7 +3054,7 @@ def fake_get_aspect(
     assert server_domain in second_domain_aspect.domains
 
 
-def test_pattern_add_container_dataset_domain_no_match(mock_datahub_graph):
+def test_pattern_add_container_dataset_domain_no_match(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     datahub_domain = builder.make_domain_urn("datahubproject.io")
     pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*"
@@ -2963,7 +3062,7 @@ def test_pattern_add_container_dataset_domain_no_match(mock_datahub_graph):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_simple_add_dataset_domain"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     def fake_get_aspect(
         entity_urn: str,
@@ -3003,10 +3102,10 @@ def fake_get_aspect(
 
 def run_pattern_dataset_schema_terms_transformation_semantics(
     semantics: TransformerSemantics,
-    mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph],
+    mock_datahub_graph_instance: DataHubGraph,
 ) -> List[RecordEnvelope]:
     pipeline_context = PipelineContext(run_id="test_pattern_dataset_schema_terms")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # fake the server response
     def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
@@ -3113,10 +3212,10 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
 
 
 def test_pattern_dataset_schema_terms_transformation_patch(
-    mock_time, mock_datahub_graph
+    mock_time, mock_datahub_graph_instance
 ):
     output = run_pattern_dataset_schema_terms_transformation_semantics(
-        TransformerSemantics.PATCH, mock_datahub_graph
+        TransformerSemantics.PATCH, mock_datahub_graph_instance
     )
     assert len(output) == 2
     # Check that glossary terms were added.
@@ -3146,10 +3245,10 @@ def test_pattern_dataset_schema_terms_transformation_patch(
 
 
 def test_pattern_dataset_schema_terms_transformation_overwrite(
-    mock_time, mock_datahub_graph
+    mock_time, mock_datahub_graph_instance
 ):
     output = run_pattern_dataset_schema_terms_transformation_semantics(
-        TransformerSemantics.OVERWRITE, mock_datahub_graph
+        TransformerSemantics.OVERWRITE, mock_datahub_graph_instance
     )
 
     assert len(output) == 2
@@ -3181,10 +3280,10 @@ def test_pattern_dataset_schema_terms_transformation_overwrite(
 
 def run_pattern_dataset_schema_tags_transformation_semantics(
     semantics: TransformerSemantics,
-    mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph],
+    mock_datahub_graph_instance: DataHubGraph,
 ) -> List[RecordEnvelope]:
     pipeline_context = PipelineContext(run_id="test_pattern_dataset_schema_terms")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # fake the server response
     def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
@@ -3284,10 +3383,10 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
 
 
 def test_pattern_dataset_schema_tags_transformation_overwrite(
-    mock_time, mock_datahub_graph
+    mock_time, mock_datahub_graph_instance
 ):
     output = run_pattern_dataset_schema_tags_transformation_semantics(
-        TransformerSemantics.OVERWRITE, mock_datahub_graph
+        TransformerSemantics.OVERWRITE, mock_datahub_graph_instance
     )
 
     assert len(output) == 2
@@ -3318,10 +3417,10 @@ def test_pattern_dataset_schema_tags_transformation_overwrite(
 
 
 def test_pattern_dataset_schema_tags_transformation_patch(
-    mock_time, mock_datahub_graph
+    mock_time, mock_datahub_graph_instance
 ):
     output = run_pattern_dataset_schema_tags_transformation_semantics(
-        TransformerSemantics.PATCH, mock_datahub_graph
+        TransformerSemantics.PATCH, mock_datahub_graph_instance
     )
 
     assert len(output) == 2
@@ -3542,9 +3641,11 @@ def fake_ownership_class(entity_urn: str) -> models.OwnershipClass:
     assert set(out_owners) == set(cleaned_owner_urn)
 
 
-def test_clean_owner_urn_transformation_remove_fixed_string(mock_datahub_graph):
+def test_clean_owner_urn_transformation_remove_fixed_string(
+    mock_datahub_graph_instance,
+):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     user_emails = [
         "ABCDEF:email_id@example.com",
@@ -3581,9 +3682,11 @@ def test_clean_owner_urn_transformation_remove_fixed_string(mock_datahub_graph):
     _test_clean_owner_urns(pipeline_context, in_owner_urns, config, expected_owner_urns)
 
 
-def test_clean_owner_urn_transformation_remove_multiple_values(mock_datahub_graph):
+def test_clean_owner_urn_transformation_remove_multiple_values(
+    mock_datahub_graph_instance,
+):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     user_emails = [
         "ABCDEF:email_id@example.com",
@@ -3620,9 +3723,11 @@ def test_clean_owner_urn_transformation_remove_multiple_values(mock_datahub_grap
     _test_clean_owner_urns(pipeline_context, in_owner_urns, config, expected_owner_urns)
 
 
-def test_clean_owner_urn_transformation_remove_values_using_regex(mock_datahub_graph):
+def test_clean_owner_urn_transformation_remove_values_using_regex(
+    mock_datahub_graph_instance,
+):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     user_emails = [
         "ABCDEF:email_id@example.com",
@@ -3659,9 +3764,9 @@ def test_clean_owner_urn_transformation_remove_values_using_regex(mock_datahub_g
     _test_clean_owner_urns(pipeline_context, in_owner_urns, config, expected_owner_urns)
 
 
-def test_clean_owner_urn_transformation_remove_digits(mock_datahub_graph):
+def test_clean_owner_urn_transformation_remove_digits(mock_datahub_graph_instance):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     user_emails = [
         "ABCDEF:email_id@example.com",
@@ -3698,9 +3803,9 @@ def test_clean_owner_urn_transformation_remove_digits(mock_datahub_graph):
     _test_clean_owner_urns(pipeline_context, in_owner_urns, config, expected_owner_urns)
 
 
-def test_clean_owner_urn_transformation_remove_pattern(mock_datahub_graph):
+def test_clean_owner_urn_transformation_remove_pattern(mock_datahub_graph_instance):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     user_emails = [
         "ABCDEF:email_id@example.com",
@@ -3738,10 +3843,10 @@ def test_clean_owner_urn_transformation_remove_pattern(mock_datahub_graph):
 
 
 def test_clean_owner_urn_transformation_remove_word_in_capital_letters(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     user_emails = [
         "ABCDEF:email_id@example.com",
@@ -3781,10 +3886,10 @@ def test_clean_owner_urn_transformation_remove_word_in_capital_letters(
 
 
 def test_clean_owner_urn_transformation_remove_pattern_with_alphanumeric_value(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     user_emails = [
         "ABCDEF:email_id@example.com",
@@ -3822,10 +3927,10 @@ def test_clean_owner_urn_transformation_remove_pattern_with_alphanumeric_value(
 
 
 def test_clean_owner_urn_transformation_should_not_remove_system_identifier(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     user_emails = [
         "ABCDEF:email_id@example.com",
@@ -3850,12 +3955,12 @@ def test_clean_owner_urn_transformation_should_not_remove_system_identifier(
 
 
 def test_replace_external_url_word_replace(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_replace_external_url"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=ReplaceExternalUrlDataset,
@@ -3877,12 +3982,12 @@ def test_replace_external_url_word_replace(
 
 
 def test_replace_external_regex_replace_1(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_replace_external_url"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=ReplaceExternalUrlDataset,
@@ -3904,12 +4009,12 @@ def test_replace_external_regex_replace_1(
 
 
 def test_replace_external_regex_replace_2(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_replace_external_url"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_dataset_transformer_pipeline(
         transformer_type=ReplaceExternalUrlDataset,
@@ -3931,12 +4036,12 @@ def test_replace_external_regex_replace_2(
 
 
 def test_pattern_cleanup_usage_statistics_user_1(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_pattern_cleanup_usage_statistics_user"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     TS_1 = datetime(year=2023, month=1, day=1, tzinfo=timezone.utc)
 
@@ -3985,12 +4090,12 @@ def test_pattern_cleanup_usage_statistics_user_1(
 
 
 def test_pattern_cleanup_usage_statistics_user_2(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_pattern_cleanup_usage_statistics_user"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     TS_1 = datetime(year=2023, month=1, day=1, tzinfo=timezone.utc)
 
@@ -4039,12 +4144,12 @@ def test_pattern_cleanup_usage_statistics_user_2(
 
 
 def test_pattern_cleanup_usage_statistics_user_3(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_pattern_cleanup_usage_statistics_user"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     TS_1 = datetime(year=2023, month=1, day=1, tzinfo=timezone.utc)
 
@@ -4092,7 +4197,7 @@ def test_pattern_cleanup_usage_statistics_user_3(
     assert output[0].record.aspect.userCounts == expectedUsageStatistics.userCounts
 
 
-def test_domain_mapping_based_on_tags_with_valid_tags(mock_datahub_graph):
+def test_domain_mapping_based_on_tags_with_valid_tags(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     server_domain = builder.make_domain_urn("test.io")
 
@@ -4103,7 +4208,7 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
         return models.GlobalTagsClass(tags=[TagAssociationClass(tag=tag_one)])
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     pipeline_context.graph.get_tags = fake_get_tags  # type: ignore
 
@@ -4126,13 +4231,15 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
     assert server_domain not in transformed_aspect.domains
 
 
-def test_domain_mapping_based_on_tags_with_no_matching_tags(mock_datahub_graph):
+def test_domain_mapping_based_on_tags_with_no_matching_tags(
+    mock_datahub_graph_instance,
+):
     acryl_domain = builder.make_domain_urn("acryl.io")
     server_domain = builder.make_domain_urn("test.io")
     non_matching_tag = builder.make_tag_urn("nonMatching")
 
     pipeline_context = PipelineContext(run_id="no_match_pipeline")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
@@ -4157,11 +4264,11 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
     assert server_domain in transformed_aspect.domains
 
 
-def test_domain_mapping_based_on_tags_with_empty_config(mock_datahub_graph):
+def test_domain_mapping_based_on_tags_with_empty_config(mock_datahub_graph_instance):
     some_tag = builder.make_tag_urn("someTag")
 
     pipeline_context = PipelineContext(run_id="empty_config_pipeline")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
@@ -4180,7 +4287,9 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
     assert len(output[0].record.aspect.domains) == 0
 
 
-def test_domain_mapping_based__r_on_tags_with_multiple_tags(mock_datahub_graph):
+def test_domain_mapping_based__r_on_tags_with_multiple_tags(
+    mock_datahub_graph_instance,
+):
     # Two tags that match different rules in the domain mapping configuration
     tag_one = builder.make_tag_urn("test:tag_1")
     tag_two = builder.make_tag_urn("test:tag_2")
@@ -4189,7 +4298,7 @@ def test_domain_mapping_based__r_on_tags_with_multiple_tags(mock_datahub_graph):
     hr = builder.make_domain_urn("hr")
 
     pipeline_context = PipelineContext(run_id="multiple_matches_pipeline")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
@@ -4226,11 +4335,11 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass:
     assert len(transformed_aspect.domains) == 3
 
 
-def test_domain_mapping_based_on_tags_with_empty_tags(mock_datahub_graph):
+def test_domain_mapping_based_on_tags_with_empty_tags(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     server_domain = builder.make_domain_urn("test.io")
     pipeline_context = PipelineContext(run_id="empty_config_pipeline")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
@@ -4254,11 +4363,11 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass:
     assert server_domain not in transformed_aspect.domains
 
 
-def test_domain_mapping_based_on_tags_with_no_tags(mock_datahub_graph):
+def test_domain_mapping_based_on_tags_with_no_tags(mock_datahub_graph_instance):
     acryl_domain = builder.make_domain_urn("acryl.io")
     server_domain = builder.make_domain_urn("test.io")
     pipeline_context = PipelineContext(run_id="empty_config_pipeline")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
 
     # Return fake aspect to simulate server behaviour
     def fake_get_tags(entity_urn: str) -> Optional[models.GlobalTagsClass]:
@@ -4282,7 +4391,7 @@ def fake_get_tags(entity_urn: str) -> Optional[models.GlobalTagsClass]:
     assert server_domain not in transformed_aspect.domains
 
 
-def test_tags_to_terms_transformation(mock_datahub_graph):
+def test_tags_to_terms_transformation(mock_datahub_graph_instance):
     # Create domain URNs for the test
     term_urn_example1 = builder.make_term_urn("example1")
     term_urn_example2 = builder.make_term_urn("example2")
@@ -4349,7 +4458,7 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
         )
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_tags = fake_get_tags  # type: ignore
     pipeline_context.graph.get_schema_metadata = fake_schema_metadata  # type: ignore
 
@@ -4383,7 +4492,7 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
     }
 
 
-def test_tags_to_terms_with_no_matching_terms(mock_datahub_graph):
+def test_tags_to_terms_with_no_matching_terms(mock_datahub_graph_instance):
     # Setup for test where no tags match the provided term mappings
     def fake_get_tags_no_match(entity_urn: str) -> models.GlobalTagsClass:
         return models.GlobalTagsClass(
@@ -4394,7 +4503,7 @@ def fake_get_tags_no_match(entity_urn: str) -> models.GlobalTagsClass:
         )
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_tags = fake_get_tags_no_match  # type: ignore
 
     # No matching terms in config
@@ -4420,13 +4529,13 @@ def fake_get_tags_no_match(entity_urn: str) -> models.GlobalTagsClass:
     assert len(terms_aspect.terms) == 1
 
 
-def test_tags_to_terms_with_missing_tags(mock_datahub_graph):
+def test_tags_to_terms_with_missing_tags(mock_datahub_graph_instance):
     # Setup for test where no tags are present
     def fake_get_no_tags(entity_urn: str) -> models.GlobalTagsClass:
         return models.GlobalTagsClass(tags=[])
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_tags = fake_get_no_tags  # type: ignore
 
     config = {"tags": ["example1", "example2"]}
@@ -4451,7 +4560,7 @@ def fake_get_no_tags(entity_urn: str) -> models.GlobalTagsClass:
     assert len(terms_aspect.terms) == 1
 
 
-def test_tags_to_terms_with_partial_match(mock_datahub_graph):
+def test_tags_to_terms_with_partial_match(mock_datahub_graph_instance):
     # Setup for partial match scenario
     def fake_get_partial_match_tags(entity_urn: str) -> models.GlobalTagsClass:
         return models.GlobalTagsClass(
@@ -4466,7 +4575,7 @@ def fake_get_partial_match_tags(entity_urn: str) -> models.GlobalTagsClass:
         )
 
     pipeline_context = PipelineContext(run_id="transformer_pipe_line")
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig())
+    pipeline_context.graph = mock_datahub_graph_instance
     pipeline_context.graph.get_tags = fake_get_partial_match_tags  # type: ignore
 
     config = {"tags": ["example1"]}  # Only 'example1' has a term mapped
@@ -4493,12 +4602,12 @@ def fake_get_partial_match_tags(entity_urn: str) -> models.GlobalTagsClass:
 
 
 def test_replace_external_url_container_word_replace(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_replace_external_url_container"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_container_transformer_pipeline(
         transformer_type=ReplaceExternalUrlContainer,
@@ -4521,12 +4630,12 @@ def test_replace_external_url_container_word_replace(
 
 
 def test_replace_external_regex_container_replace_1(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_replace_external_url_container"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_container_transformer_pipeline(
         transformer_type=ReplaceExternalUrlContainer,
@@ -4549,12 +4658,12 @@ def test_replace_external_regex_container_replace_1(
 
 
 def test_replace_external_regex_container_replace_2(
-    mock_datahub_graph,
+    mock_datahub_graph_instance,
 ):
     pipeline_context: PipelineContext = PipelineContext(
         run_id="test_replace_external_url_container"
     )
-    pipeline_context.graph = mock_datahub_graph(DatahubClientConfig)
+    pipeline_context.graph = mock_datahub_graph_instance
 
     output = run_container_transformer_pipeline(
         transformer_type=ReplaceExternalUrlContainer,
diff --git a/metadata-ingestion/tests/unit/urns/invalid_urns.txt b/metadata-ingestion/tests/unit/urns/invalid_urns.txt
new file mode 100644
index 0000000000000..9ce2c99a1a4ee
--- /dev/null
+++ b/metadata-ingestion/tests/unit/urns/invalid_urns.txt
@@ -0,0 +1,40 @@
+# Basic URN format tests
+urn:li:abc
+urn:li:abc:
+urn:li:abc:()
+urn:li:abc:(abc,)
+urn:li:corpuser:abc)
+
+# Reserved characters
+urn:li:corpuser:foo␟bar
+urn:li:tag:a,b,c
+
+# CorpUser URN tests
+urn:li:corpuser:(part1,part2)
+
+# Dataset URN tests
+urn:li:dataset:(urn:li:user:abc,dataset,prod)
+urn:li:dataset:(urn:li:user:abc,dataset)
+urn:li:dataset:(urn:li:user:abc,dataset,invalidEnv)
+
+# DataFlow URN tests
+urn:li:dataFlow:(airflow,flow_id)
+
+# DataJob URN tests
+urn:li:dataJob:(urn:li:user:abc,job_id)
+urn:li:dataJob:(urn:li:dataFlow:(airflow,flow_id,prod))
+
+# Domain URN tests
+urn:li:domain:(part1,part2)
+
+# Tag URN tests
+urn:li:tag:(part1,part2)
+
+# Notebook URN tests
+urn:li:notebook:(part1,part2,part3)
+
+# CorpGroup URN tests
+urn:li:corpGroup:(part1,part2)
+
+# DataProcessInstance URN tests
+urn:li:dataProcessInstance:(part1,part2)
diff --git a/metadata-ingestion/tests/unit/urns/test_corp_group_urn.py b/metadata-ingestion/tests/unit/urns/test_corp_group_urn.py
index 1897a0e8686f0..4e55e78255d1c 100644
--- a/metadata-ingestion/tests/unit/urns/test_corp_group_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_corp_group_urn.py
@@ -3,7 +3,6 @@
 import pytest
 
 from datahub.utilities.urns.corp_group_urn import CorpGroupUrn
-from datahub.utilities.urns.error import InvalidUrnError
 
 
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")
@@ -17,12 +16,3 @@ def test_parse_urn(self) -> None:
         assert str(corp_group_urn) == corp_group_urn_str
         assert corp_group_urn == CorpGroupUrn(name="abc")
         assert corp_group_urn == CorpGroupUrn.create_from_id("abc")
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            CorpGroupUrn.create_from_string(
-                "urn:li:abc:(urn:li:dataPlatform:abc,def,prod)"
-            )
-
-        with self.assertRaises(InvalidUrnError):
-            CorpGroupUrn.create_from_string("urn:li:corpGroup:(part1,part2)")
diff --git a/metadata-ingestion/tests/unit/urns/test_corpuser_urn.py b/metadata-ingestion/tests/unit/urns/test_corpuser_urn.py
index 7a2a4f4ff4493..e4a11b4f404c6 100644
--- a/metadata-ingestion/tests/unit/urns/test_corpuser_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_corpuser_urn.py
@@ -3,7 +3,6 @@
 import pytest
 
 from datahub.utilities.urns.corpuser_urn import CorpuserUrn
-from datahub.utilities.urns.error import InvalidUrnError
 
 
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")
@@ -17,12 +16,3 @@ def test_parse_urn(self) -> None:
         assert str(corpuser_urn) == corpuser_urn_str
         assert corpuser_urn == CorpuserUrn("abc")
         assert corpuser_urn == CorpuserUrn.create_from_id("abc")
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            CorpuserUrn.create_from_string(
-                "urn:li:abc:(urn:li:dataPlatform:abc,def,prod)"
-            )
-
-        with self.assertRaises(InvalidUrnError):
-            CorpuserUrn.create_from_string("urn:li:corpuser:(part1,part2)")
diff --git a/metadata-ingestion/tests/unit/urns/test_data_flow_urn.py b/metadata-ingestion/tests/unit/urns/test_data_flow_urn.py
index 524411121d418..edb5563c5b22e 100644
--- a/metadata-ingestion/tests/unit/urns/test_data_flow_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_data_flow_urn.py
@@ -3,7 +3,6 @@
 import pytest
 
 from datahub.utilities.urns.data_flow_urn import DataFlowUrn
-from datahub.utilities.urns.error import InvalidUrnError
 
 
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")
@@ -16,10 +15,3 @@ def test_parse_urn(self) -> None:
         assert data_flow_urn.get_env() == "prod"
         assert data_flow_urn.__str__() == "urn:li:dataFlow:(airflow,def,prod)"
         assert data_flow_urn == DataFlowUrn("airflow", "def", "prod")
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            DataFlowUrn.create_from_string("urn:li:abc:(airflow,def,prod)")
-
-        with self.assertRaises(InvalidUrnError):
-            DataFlowUrn.create_from_string("urn:li:dataFlow:(airflow,flow_id)")
diff --git a/metadata-ingestion/tests/unit/urns/test_data_job_urn.py b/metadata-ingestion/tests/unit/urns/test_data_job_urn.py
index bf039cd2a91f9..484e5a474c0cd 100644
--- a/metadata-ingestion/tests/unit/urns/test_data_job_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_data_job_urn.py
@@ -4,7 +4,6 @@
 
 from datahub.utilities.urns.data_flow_urn import DataFlowUrn
 from datahub.utilities.urns.data_job_urn import DataJobUrn
-from datahub.utilities.urns.error import InvalidUrnError
 
 
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")
@@ -22,17 +21,3 @@ def test_parse_urn(self) -> None:
         assert data_job_urn == DataJobUrn(
             "urn:li:dataFlow:(airflow,flow_id,prod)", "job_id"
         )
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            DataJobUrn.create_from_string(
-                "urn:li:abc:(urn:li:dataFlow:(airflow,flow_id,prod),job_id)"
-            )
-
-        with self.assertRaises(InvalidUrnError):
-            DataJobUrn.create_from_string("urn:li:dataJob:(urn:li:user:abc,job_id)")
-
-        with self.assertRaises(InvalidUrnError):
-            DataJobUrn.create_from_string(
-                "urn:li:dataJob:(urn:li:dataFlow:(airflow,flow_id,prod))"
-            )
diff --git a/metadata-ingestion/tests/unit/urns/test_data_process_instance_urn.py b/metadata-ingestion/tests/unit/urns/test_data_process_instance_urn.py
index a86f8dd99416f..f9087b19b13c3 100644
--- a/metadata-ingestion/tests/unit/urns/test_data_process_instance_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_data_process_instance_urn.py
@@ -3,7 +3,6 @@
 import pytest
 
 from datahub.utilities.urns.data_process_instance_urn import DataProcessInstanceUrn
-from datahub.utilities.urns.error import InvalidUrnError
 
 
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")
@@ -20,12 +19,3 @@ def test_parse_urn(self) -> None:
         assert dataprocessinstance_urn == DataProcessInstanceUrn("abc")
         assert dataprocessinstance_urn == DataProcessInstanceUrn.create_from_id("abc")
         assert "abc" == dataprocessinstance_urn.get_dataprocessinstance_id()
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            DataProcessInstanceUrn.create_from_string("urn:li:abc:dataProcessInstance")
-
-        with self.assertRaises(InvalidUrnError):
-            DataProcessInstanceUrn.create_from_string(
-                "urn:li:dataProcessInstance:(part1,part2)"
-            )
diff --git a/metadata-ingestion/tests/unit/urns/test_dataset_urn.py b/metadata-ingestion/tests/unit/urns/test_dataset_urn.py
index 53065143a6ae4..1be5cd5915200 100644
--- a/metadata-ingestion/tests/unit/urns/test_dataset_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_dataset_urn.py
@@ -4,7 +4,6 @@
 
 from datahub.utilities.urns.data_platform_urn import DataPlatformUrn
 from datahub.utilities.urns.dataset_urn import DatasetUrn
-from datahub.utilities.urns.error import InvalidUrnError
 
 
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")
@@ -20,22 +19,3 @@ def test_parse_urn(self) -> None:
         assert dataset_urn.get_env() == "PROD"
         assert dataset_urn.__str__() == dataset_urn_str
         assert dataset_urn == DatasetUrn("urn:li:dataPlatform:abc", "def", "prod")
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            DatasetUrn.create_from_string(
-                "urn:li:abc:(urn:li:dataPlatform:abc,def,prod)"
-            )
-
-        with self.assertRaises(InvalidUrnError):
-            DatasetUrn.create_from_string(
-                "urn:li:dataset:(urn:li:user:abc,dataset,prod)"
-            )
-
-        with self.assertRaises(InvalidUrnError):
-            DatasetUrn.create_from_string("urn:li:dataset:(urn:li:user:abc,dataset)")
-
-        with self.assertRaises(InvalidUrnError):
-            DatasetUrn.create_from_string(
-                "urn:li:dataset:(urn:li:user:abc,dataset,invalidEnv)"
-            )
diff --git a/metadata-ingestion/tests/unit/urns/test_domain_urn.py b/metadata-ingestion/tests/unit/urns/test_domain_urn.py
index 843a5bf40f5c6..aa5050ce1c030 100644
--- a/metadata-ingestion/tests/unit/urns/test_domain_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_domain_urn.py
@@ -3,7 +3,6 @@
 import pytest
 
 from datahub.utilities.urns.domain_urn import DomainUrn
-from datahub.utilities.urns.error import InvalidUrnError
 
 
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")
@@ -17,10 +16,3 @@ def test_parse_urn(self) -> None:
         assert str(domain_urn) == domain_urn_str
         assert domain_urn == DomainUrn("abc")
         assert domain_urn == DomainUrn.create_from_id("abc")
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            DomainUrn.create_from_string("urn:li:abc:domain")
-
-        with self.assertRaises(InvalidUrnError):
-            DomainUrn.create_from_string("urn:li:domain:(part1,part2)")
diff --git a/metadata-ingestion/tests/unit/urns/test_notebook_urn.py b/metadata-ingestion/tests/unit/urns/test_notebook_urn.py
index 3ec580f02142b..6d4dd2ee6fa8c 100644
--- a/metadata-ingestion/tests/unit/urns/test_notebook_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_notebook_urn.py
@@ -2,7 +2,6 @@
 
 import pytest
 
-from datahub.utilities.urns.error import InvalidUrnError
 from datahub.utilities.urns.notebook_urn import NotebookUrn
 
 
@@ -16,12 +15,3 @@ def test_parse_urn(self) -> None:
         assert str(notebook_urn) == notebook_urn_str
 
         assert notebook_urn == NotebookUrn("querybook", "123")
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            NotebookUrn.create_from_string(
-                "urn:li:abc:(urn:li:dataPlatform:abc,def,prod)"
-            )
-
-        with self.assertRaises(InvalidUrnError):
-            NotebookUrn.create_from_string("urn:li:notebook:(part1,part2,part3)")
diff --git a/metadata-ingestion/tests/unit/urns/test_tag_urn.py b/metadata-ingestion/tests/unit/urns/test_tag_urn.py
index fa3664bcc0218..5f4c9077e2829 100644
--- a/metadata-ingestion/tests/unit/urns/test_tag_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_tag_urn.py
@@ -2,7 +2,6 @@
 
 import pytest
 
-from datahub.utilities.urns.error import InvalidUrnError
 from datahub.utilities.urns.tag_urn import TagUrn
 
 
@@ -17,10 +16,3 @@ def test_parse_urn(self) -> None:
         assert str(tag_urn) == tag_urn_str
         assert tag_urn == TagUrn("abc")
         assert tag_urn == TagUrn.create_from_id("abc")
-
-    def test_invalid_urn(self) -> None:
-        with self.assertRaises(InvalidUrnError):
-            TagUrn.create_from_string("urn:li:abc:tag_id")
-
-        with self.assertRaises(InvalidUrnError):
-            TagUrn.create_from_string("urn:li:tag:(part1,part2)")
diff --git a/metadata-ingestion/tests/unit/urns/test_urn.py b/metadata-ingestion/tests/unit/urns/test_urn.py
index 1bf48082fec8c..0c362473c0cf1 100644
--- a/metadata-ingestion/tests/unit/urns/test_urn.py
+++ b/metadata-ingestion/tests/unit/urns/test_urn.py
@@ -1,10 +1,17 @@
+import logging
+import pathlib
+from typing import List
+
 import pytest
 
-from datahub.metadata.urns import DatasetUrn, Urn
+from datahub.metadata.urns import CorpUserUrn, DatasetUrn, Urn
 from datahub.utilities.urns.error import InvalidUrnError
 
 pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning")
 
+_CURRENT_DIR = pathlib.Path(__file__).parent
+logger = logging.getLogger(__name__)
+
 
 def test_parse_urn() -> None:
     simple_urn_str = "urn:li:dataPlatform:abc"
@@ -34,23 +41,67 @@ def test_url_encode_urn() -> None:
     )
 
 
-def test_invalid_urn() -> None:
-    with pytest.raises(InvalidUrnError):
-        Urn.create_from_string("urn:li:abc")
+def test_urn_colon() -> None:
+    # There's a bunch of other, simpler tests for special characters in the valid_urns test.
+
+    # This test ensures that the type dispatch and fields work fine here.
+    # I'm not sure why you'd ever want this, but technically it's a valid urn.
+
+    urn = Urn.from_string("urn:li:corpuser::")
+    assert isinstance(urn, CorpUserUrn)
+    assert urn.username == ":"
+    assert urn == CorpUserUrn(":")
 
-    with pytest.raises(InvalidUrnError):
-        Urn.create_from_string("urn:li:abc:")
 
-    with pytest.raises(InvalidUrnError):
-        Urn.create_from_string("urn:li:abc:()")
+def test_urn_coercion() -> None:
+    urn = CorpUserUrn("foo␟bar")
+    assert urn.urn() == "urn:li:corpuser:foo%E2%90%9Fbar"
 
-    with pytest.raises(InvalidUrnError):
-        Urn.create_from_string("urn:li:abc:(abc,)")
+    assert urn == Urn.from_string(urn.urn())
 
 
-def test_urn_type_dispatch() -> None:
-    urn = Urn.from_string("urn:li:dataset:(urn:li:dataPlatform:abc,def,prod)")
+def test_urn_type_dispatch_1() -> None:
+    urn = Urn.from_string("urn:li:dataset:(urn:li:dataPlatform:abc,def,PROD)")
     assert isinstance(urn, DatasetUrn)
 
     with pytest.raises(InvalidUrnError, match="Passed an urn of type corpuser"):
         DatasetUrn.from_string("urn:li:corpuser:foo")
+
+
+def test_urn_type_dispatch_2() -> None:
+    urn = "urn:li:dataJob:(urn:li:dataFlow:(airflow,flow_id,prod),job_id)"
+    assert Urn.from_string(urn).urn() == urn
+
+    with pytest.raises(InvalidUrnError, match="Passed an urn of type dataJob"):
+        CorpUserUrn.from_string(urn)
+
+
+def _load_urns(file_name: pathlib.Path) -> List[str]:
+    urns = [
+        line.strip()
+        for line in file_name.read_text().splitlines()
+        if line.strip() and not line.startswith("#")
+    ]
+    assert len(urns) > 0, f"No urns found in {file_name}"
+    return urns
+
+
+def test_valid_urns() -> None:
+    valid_urns_file = _CURRENT_DIR / "valid_urns.txt"
+    valid_urns = _load_urns(valid_urns_file)
+
+    for valid_urn in valid_urns:
+        logger.info(f"Testing valid URN: {valid_urn}")
+        parsed_urn = Urn.from_string(valid_urn)
+        assert parsed_urn.urn() == valid_urn
+
+
+def test_invalid_urns() -> None:
+    invalid_urns_file = _CURRENT_DIR / "invalid_urns.txt"
+    invalid_urns = _load_urns(invalid_urns_file)
+
+    # Test each invalid URN
+    for invalid_urn in invalid_urns:
+        with pytest.raises(InvalidUrnError):
+            logger.info(f"Testing invalid URN: {invalid_urn}")
+            Urn.from_string(invalid_urn)
diff --git a/metadata-ingestion/tests/unit/urns/valid_urns.txt b/metadata-ingestion/tests/unit/urns/valid_urns.txt
new file mode 100644
index 0000000000000..23205ec9a7235
--- /dev/null
+++ b/metadata-ingestion/tests/unit/urns/valid_urns.txt
@@ -0,0 +1,24 @@
+# Unknown entity types become generic urns
+urn:li:abc:foo
+urn:li:abc:(foo,bar)
+urn:li:abc:(urn:li:dataPlatform:abc,def,prod)
+
+# A bunch of pretty normal urns
+urn:li:corpuser:foo
+urn:li:corpGroup:bar
+urn:li:dataset:(urn:li:dataPlatform:abc,def/ghi,prod)
+urn:li:dataFlow:(airflow,def,prod)
+urn:li:dataJob:(urn:li:dataFlow:(airflow,flow_id,prod),job_id)
+urn:li:tag:abc
+urn:li:chart:(looker,chart_name)
+urn:li:dashboard:(looker,dashboard_name)
+urn:li:dataProcessInstance:abc
+urn:li:domain:abc
+urn:li:notebook:(querybook,123)
+
+# Urns with colons and other special characters
+urn:li:tag:dbt:bar
+urn:li:tag::
+urn:li:dashboard:(looker,dashboards.thelook::customer_lookup)
+urn:li:dataPlatform:abc:def
+urn:li:corpuser:foo:bar@example.com
diff --git a/metadata-ingestion/tests/unit/test_cli_logging.py b/metadata-ingestion/tests/unit/utilities/test_cli_logging.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_cli_logging.py
rename to metadata-ingestion/tests/unit/utilities/test_cli_logging.py
diff --git a/metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py b/metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py
index f4062f9a91145..6230c2e37edc6 100644
--- a/metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py
+++ b/metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py
@@ -15,11 +15,13 @@
 )
 
 
-def test_file_dict() -> None:
+@pytest.mark.parametrize("use_sqlite_on_conflict", [True, False])
+def test_file_dict(use_sqlite_on_conflict: bool) -> None:
     cache = FileBackedDict[int](
         tablename="cache",
         cache_max_size=10,
         cache_eviction_batch_size=10,
+        _use_sqlite_on_conflict=use_sqlite_on_conflict,
     )
 
     for i in range(100):
@@ -92,7 +94,8 @@ def test_file_dict() -> None:
         cache["a"] = 1
 
 
-def test_custom_serde() -> None:
+@pytest.mark.parametrize("use_sqlite_on_conflict", [True, False])
+def test_custom_serde(use_sqlite_on_conflict: bool) -> None:
     @dataclass(frozen=True)
     class Label:
         a: str
@@ -139,6 +142,7 @@ def deserialize(s: str) -> Main:
         deserializer=deserialize,
         # Disable the in-memory cache to force all reads/writes to the DB.
         cache_max_size=0,
+        _use_sqlite_on_conflict=use_sqlite_on_conflict,
     )
     first = Main(3, {Label("one", 1): 0.1, Label("two", 2): 0.2})
     second = Main(-100, {Label("z", 26): 0.26})
@@ -186,7 +190,8 @@ def test_file_dict_stores_counter() -> None:
         assert in_memory_counters[i].most_common(2) == cache[str(i)].most_common(2)
 
 
-def test_file_dict_ordering() -> None:
+@pytest.mark.parametrize("use_sqlite_on_conflict", [True, False])
+def test_file_dict_ordering(use_sqlite_on_conflict: bool) -> None:
     """
     We require that FileBackedDict maintains insertion order, similar to Python's
     built-in dict. This test makes one of each and validates that they behave the same.
@@ -196,6 +201,7 @@ def test_file_dict_ordering() -> None:
         serializer=str,
         deserializer=int,
         cache_max_size=1,
+        _use_sqlite_on_conflict=use_sqlite_on_conflict,
     )
     data = {}
 
@@ -229,12 +235,14 @@ class Pair:
 
 
 @pytest.mark.parametrize("cache_max_size", [0, 1, 10])
-def test_custom_column(cache_max_size: int) -> None:
+@pytest.mark.parametrize("use_sqlite_on_conflict", [True, False])
+def test_custom_column(cache_max_size: int, use_sqlite_on_conflict: bool) -> None:
     cache = FileBackedDict[Pair](
         extra_columns={
             "x": lambda m: m.x,
         },
         cache_max_size=cache_max_size,
+        _use_sqlite_on_conflict=use_sqlite_on_conflict,
     )
 
     cache["first"] = Pair(3, "a")
@@ -275,7 +283,8 @@ def test_custom_column(cache_max_size: int) -> None:
     ]
 
 
-def test_shared_connection() -> None:
+@pytest.mark.parametrize("use_sqlite_on_conflict", [True, False])
+def test_shared_connection(use_sqlite_on_conflict: bool) -> None:
     with ConnectionWrapper() as connection:
         cache1 = FileBackedDict[int](
             shared_connection=connection,
@@ -283,6 +292,7 @@ def test_shared_connection() -> None:
             extra_columns={
                 "v": lambda v: v,
             },
+            _use_sqlite_on_conflict=use_sqlite_on_conflict,
         )
         cache2 = FileBackedDict[Pair](
             shared_connection=connection,
@@ -291,6 +301,7 @@ def test_shared_connection() -> None:
                 "x": lambda m: m.x,
                 "y": lambda m: m.y,
             },
+            _use_sqlite_on_conflict=use_sqlite_on_conflict,
         )
 
         cache1["a"] = 3
diff --git a/metadata-ingestion/tests/unit/test_ordered_set.py b/metadata-ingestion/tests/unit/utilities/test_ordered_set.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_ordered_set.py
rename to metadata-ingestion/tests/unit/utilities/test_ordered_set.py
diff --git a/metadata-ingestion/tests/unit/test_parsing_util.py b/metadata-ingestion/tests/unit/utilities/test_parsing_util.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_parsing_util.py
rename to metadata-ingestion/tests/unit/utilities/test_parsing_util.py
diff --git a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py
index 2901b4bba107e..ce211c2d61806 100644
--- a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py
+++ b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py
@@ -1,7 +1,11 @@
 import logging
+import math
 import time
 from concurrent.futures import Future
 
+import pytest
+from pydantic.schema import timedelta
+
 from datahub.utilities.partition_executor import (
     BatchPartitionExecutor,
     PartitionExecutor,
@@ -76,7 +80,8 @@ def task(id: str) -> str:
         assert len(done_tasks) == 16
 
 
-def test_batch_partition_executor_sequential_key_execution():
+@pytest.mark.parametrize("max_workers", [1, 2, 10])
+def test_batch_partition_executor_sequential_key_execution(max_workers: int) -> None:
     executing_tasks = set()
     done_tasks = set()
     done_task_batches = set()
@@ -95,7 +100,7 @@ def process_batch(batch):
         done_task_batches.add(tuple(id for _, id in batch))
 
     with BatchPartitionExecutor(
-        max_workers=2,
+        max_workers=max_workers,
         max_pending=10,
         max_per_batch=2,
         process_batch=process_batch,
@@ -129,7 +134,9 @@ def process_batch(batch):
     }
 
 
+@pytest.mark.timeout(5)
 def test_batch_partition_executor_max_batch_size():
+    n = 5
     batches_processed = []
 
     def process_batch(batch):
@@ -137,19 +144,52 @@ def process_batch(batch):
         time.sleep(0.1)  # Simulate batch processing time
 
     with BatchPartitionExecutor(
-        max_workers=5, max_pending=20, process_batch=process_batch, max_per_batch=2
+        max_workers=5,
+        max_pending=10,
+        process_batch=process_batch,
+        max_per_batch=2,
+        min_process_interval=timedelta(seconds=0.1),
+        read_from_pending_interval=timedelta(seconds=0.1),
     ) as executor:
         # Submit more tasks than the max_per_batch to test batching limits.
-        for i in range(5):
+        for i in range(n):
             executor.submit("key3", "key3", f"task{i}")
 
     # Check the batches.
     logger.info(f"batches_processed: {batches_processed}")
-    assert len(batches_processed) == 3
+    assert len(batches_processed) == math.ceil(n / 2), "Incorrect number of batches"
     for batch in batches_processed:
         assert len(batch) <= 2, "Batch size exceeded max_per_batch limit"
 
 
+@pytest.mark.timeout(10)
+def test_batch_partition_executor_deadlock():
+    n = 20  # Exceed max_pending to test for deadlocks when max_pending exceeded
+    batch_size = 2
+    batches_processed = []
+
+    def process_batch(batch):
+        batches_processed.append(batch)
+        time.sleep(0.1)  # Simulate batch processing time
+
+    with BatchPartitionExecutor(
+        max_workers=5,
+        max_pending=2,
+        process_batch=process_batch,
+        max_per_batch=batch_size,
+        min_process_interval=timedelta(seconds=30),
+        read_from_pending_interval=timedelta(seconds=0.01),
+    ) as executor:
+        # Submit more tasks than the max_per_batch to test batching limits.
+        executor.submit("key3", "key3", "task0")
+        executor.submit("key3", "key3", "task1")
+        executor.submit("key1", "key1", "task1")  # Populates second batch
+        for i in range(3, n):
+            executor.submit("key3", "key3", f"task{i}")
+
+    assert sum(len(batch) for batch in batches_processed) == n
+
+
 def test_empty_batch_partition_executor():
     # We want to test that even if no submit() calls are made, cleanup works fine.
     with BatchPartitionExecutor(
diff --git a/metadata-ingestion/tests/unit/utilities/test_perf_timer.py b/metadata-ingestion/tests/unit/utilities/test_perf_timer.py
index 6129b3e37d8bc..166e40ef54308 100644
--- a/metadata-ingestion/tests/unit/utilities/test_perf_timer.py
+++ b/metadata-ingestion/tests/unit/utilities/test_perf_timer.py
@@ -8,39 +8,53 @@
 approx = partial(pytest.approx, rel=2e-2)
 
 
-def test_perf_timer_simple():
+def test_perf_timer_simple() -> None:
     with PerfTimer() as timer:
-        time.sleep(1)
-        assert approx(timer.elapsed_seconds()) == 1
+        time.sleep(0.4)
+        assert approx(timer.elapsed_seconds()) == 0.4
 
-    assert approx(timer.elapsed_seconds()) == 1
+    assert approx(timer.elapsed_seconds()) == 0.4
 
 
-def test_perf_timer_paused_timer():
+def test_perf_timer_paused_timer() -> None:
     with PerfTimer() as current_timer:
-        time.sleep(1)
-        assert approx(current_timer.elapsed_seconds()) == 1
+        time.sleep(0.5)
+        assert approx(current_timer.elapsed_seconds()) == 0.5
         with current_timer.pause():
-            time.sleep(2)
-            assert approx(current_timer.elapsed_seconds()) == 1
-        assert approx(current_timer.elapsed_seconds()) == 1
-        time.sleep(1)
+            time.sleep(0.3)
+            assert approx(current_timer.elapsed_seconds()) == 0.5
+        assert approx(current_timer.elapsed_seconds()) == 0.5
+        time.sleep(0.2)
 
-    assert approx(current_timer.elapsed_seconds()) == 2
+    assert approx(current_timer.elapsed_seconds()) == 0.7
 
 
-def test_generator_with_paused_timer():
+def test_generator_with_paused_timer() -> None:
+    n = 4
+
     def generator_function():
         with PerfTimer() as inner_timer:
             time.sleep(1)
-            for i in range(10):
+            for i in range(n):
                 time.sleep(0.2)
                 with inner_timer.pause():
                     time.sleep(0.2)
                     yield i
-            assert approx(inner_timer.elapsed_seconds()) == 1 + 0.2 * 10
+            assert approx(inner_timer.elapsed_seconds()) == 1 + 0.2 * n
 
     with PerfTimer() as outer_timer:
         seq = generator_function()
         list([i for i in seq])
-        assert approx(outer_timer.elapsed_seconds()) == 1 + 0.2 * 10 + 0.2 * 10
+        assert approx(outer_timer.elapsed_seconds()) == 1 + 0.2 * n + 0.2 * n
+
+
+def test_perf_timer_reuse() -> None:
+    timer = PerfTimer()
+
+    with timer:
+        time.sleep(0.2)
+
+    with timer:
+        time.sleep(0.3)
+
+    assert approx(timer.elapsed_seconds()) == 0.5
diff --git a/metadata-ingestion/tests/unit/utilities/test_progress_timer.py b/metadata-ingestion/tests/unit/utilities/test_progress_timer.py
new file mode 100644
index 0000000000000..139bad371bb9f
--- /dev/null
+++ b/metadata-ingestion/tests/unit/utilities/test_progress_timer.py
@@ -0,0 +1,53 @@
+from datetime import timedelta
+from time import sleep
+
+from datahub.utilities.progress_timer import ProgressTimer
+
+
+def test_progress_timer_basic():
+    timer = ProgressTimer(report_every=timedelta(milliseconds=100))
+
+    # First call should not report since report_0=False by default
+    assert not timer.should_report()
+
+    # Call before interval elapsed should not report
+    sleep(0.05)  # 50ms
+    assert not timer.should_report()
+
+    # Call after interval elapsed should report
+    sleep(0.1)  # Additional 100ms
+    assert timer.should_report()
+
+    # Next immediate call should not report
+    assert not timer.should_report()
+
+
+def test_progress_timer_with_report_0():
+    timer = ProgressTimer(report_every=timedelta(milliseconds=100), report_0=True)
+
+    # First call should report since report_0=True
+    assert timer.should_report()
+
+    # Next immediate call should not report
+    assert not timer.should_report()
+
+    # Call after interval elapsed should report
+    sleep(0.1)  # 100ms
+    assert timer.should_report()
+
+
+def test_progress_timer_multiple_intervals():
+    timer = ProgressTimer(report_every=timedelta(milliseconds=50))
+
+    # First call should not report
+    assert not timer.should_report()
+
+    # Check multiple intervals
+    sleep(0.06)  # 60ms - should report
+    assert timer.should_report()
+
+    sleep(0.02)  # 20ms - should not report
+    assert not timer.should_report()
+
+    sleep(0.05)  # 50ms - should report
+    assert timer.should_report()
diff --git a/metadata-ingestion/tests/unit/utilities/test_search_utils.py b/metadata-ingestion/tests/unit/utilities/test_search_utils.py
new file mode 100644
index 0000000000000..6fa2e46c7f20e
--- /dev/null
+++ b/metadata-ingestion/tests/unit/utilities/test_search_utils.py
@@ -0,0 +1,71 @@
+from datahub.utilities.search_utils import (
+    ElasticDocumentQuery,
+    LogicalOperator,
+    SearchField,
+)
+
+
+def test_simple_and_filters():
+    query = (
+        ElasticDocumentQuery.create_from()
+        .group(LogicalOperator.AND)
+        .add_field_match("field1", "value1")
+        .add_field_match("field2", "value2")
+        .end()
+    )
+
+    assert query.build() == '(field1:"value1" AND field2:"value2")'
+
+
+def test_simple_or_filters():
+    query = (
+        ElasticDocumentQuery.create_from()
+        .group(LogicalOperator.OR)
+        .add_field_match("field1", "value1")
+        .add_field_match("field2", "value2")
+        .end()
+    )
+
+    assert query.build() == '(field1:"value1" OR field2:"value2")'
+
+    # Use SearchFilter to create this query
+    query = (
+        ElasticDocumentQuery.create_from()
+        .group(LogicalOperator.OR)
+        .add_field_match(SearchField.from_string_field("field1"), "value1")
+        .add_field_match(SearchField.from_string_field("field2"), "value2")
+        .end()
+    )
+    assert query.build() == '(field1:"value1" OR field2:"value2")'
+
+
+def test_simple_field_match():
+    query: ElasticDocumentQuery = ElasticDocumentQuery.create_from(
+        ("field1", "value1:1")
+    )
+    assert query.build() == 'field1:"value1\\:1"'
+
+    # Another way to create the same query
+    query = ElasticDocumentQuery.create_from()
+    query.add_field_match("field1", "value1:1")
+    assert query.build() == 'field1:"value1\\:1"'
+
+
+def test_negation():
+    query = (
+        ElasticDocumentQuery.create_from()
+        .group(LogicalOperator.AND)
+        .add_field_match("field1", "value1")
+        .add_field_not_match("field2", "value2")
+        .end()
+    )
+
+    assert query.build() == '(field1:"value1" AND -field2:"value2")'
+
+
+def test_multi_arg_create_from():
+    query: ElasticDocumentQuery = ElasticDocumentQuery.create_from(
+        ("field1", "value1"),
+        ("field2", "value2"),
+    )
+    assert query.build() == '(field1:"value1" AND field2:"value2")'
diff --git a/metadata-ingestion/tests/unit/test_serialized_lru_cache.py b/metadata-ingestion/tests/unit/utilities/test_serialized_lru_cache.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_serialized_lru_cache.py
rename to metadata-ingestion/tests/unit/utilities/test_serialized_lru_cache.py
diff --git a/metadata-ingestion/tests/unit/utilities/test_threading_timeout.py b/metadata-ingestion/tests/unit/utilities/test_threading_timeout.py
new file mode 100644
index 0000000000000..c52d18bdd55c2
--- /dev/null
+++ b/metadata-ingestion/tests/unit/utilities/test_threading_timeout.py
@@ -0,0 +1,31 @@
+import time
+
+import pytest
+
+from datahub.utilities.threading_timeout import TimeoutException, threading_timeout
+
+
+def test_timeout_no_timeout():
+    # Should complete without raising an exception
+    with threading_timeout(1.0):
+        time.sleep(0.1)
+
+
+def test_timeout_raises():
+    # Should raise TimeoutException
+    with pytest.raises(TimeoutException):
+        with threading_timeout(0.1):
+            time.sleep(0.5)
+
+
+def test_timeout_early_exit():
+    # Test that context manager handles other exceptions properly
+    with pytest.raises(ValueError):
+        with threading_timeout(1.0):
+            raise ValueError("Early exit")
+
+
+def test_timeout_zero():
+    # Should not raise an exception
+    with threading_timeout(0.0):
+        pass
diff --git a/metadata-ingestion/tests/unit/test_topological_sort.py b/metadata-ingestion/tests/unit/utilities/test_topological_sort.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_topological_sort.py
rename to metadata-ingestion/tests/unit/utilities/test_topological_sort.py
diff --git a/metadata-ingestion/tests/unit/utilities/test_unified_diff.py b/metadata-ingestion/tests/unit/utilities/test_unified_diff.py
new file mode 100644
index 0000000000000..05277ec3fa0ab
--- /dev/null
+++ b/metadata-ingestion/tests/unit/utilities/test_unified_diff.py
@@ -0,0 +1,191 @@
+import pytest
+
+from datahub.utilities.unified_diff import (
+    DiffApplyError,
+    Hunk,
+    InvalidDiffError,
+    apply_diff,
+    apply_hunk,
+    find_hunk_start,
+    parse_patch,
+)
+
+
+def test_parse_patch():
+    patch_text = """@@ -1,3 +1,4 @@
+ Line 1
+-Line 2
++Line 2 modified
++Line 2.5
+ Line 3"""
+    hunks = parse_patch(patch_text)
+    assert len(hunks) == 1
+    assert hunks[0].source_start == 1
+    assert hunks[0].source_lines == 3
+    assert hunks[0].target_start == 1
+    assert hunks[0].target_lines == 4
+    assert hunks[0].lines == [
+        (" ", "Line 1"),
+        ("-", "Line 2"),
+        ("+", "Line 2 modified"),
+        ("+", "Line 2.5"),
+        (" ", "Line 3"),
+    ]
+
+
+def test_parse_patch_invalid():
+    with pytest.raises(InvalidDiffError):
+        parse_patch("Invalid patch")
+
+
+def test_parse_patch_bad_header():
+    # A patch with a malformed header
+    bad_patch_text = """@@ -1,3
+ Line 1
+-Line 2
++Line 2 modified
+ Line 3"""
+    with pytest.raises(InvalidDiffError):
+        parse_patch(bad_patch_text)
+
+
+def test_find_hunk_start():
+    source_lines = ["Line 1", "Line 2", "Line 3", "Line 4"]
+    hunk = Hunk(2, 2, 2, 2, [(" ", "Line 2"), (" ", "Line 3")])
+    assert find_hunk_start(source_lines, hunk) == 1
+
+
+def test_find_hunk_start_not_found():
+    source_lines = ["Line 1", "Line 2", "Line 3", "Line 4"]
+    hunk = Hunk(2, 2, 2, 2, [(" ", "Line X"), (" ", "Line Y")])
+    with pytest.raises(DiffApplyError, match="Could not find match for hunk context."):
+        find_hunk_start(source_lines, hunk)
+
+
+def test_apply_hunk_success():
+    result_lines = ["Line 1", "Line 2", "Line 3"]
+    hunk = Hunk(
+        2,
+        2,
+        2,
+        3,
+        [(" ", "Line 2"), ("-", "Line 3"), ("+", "Line 3 modified"), ("+", "Line 3.5")],
+    )
+    apply_hunk(result_lines, hunk, 0)
+    assert result_lines == ["Line 1", "Line 2", "Line 3 modified", "Line 3.5"]
+
+
+def test_apply_hunk_mismatch():
+    result_lines = ["Line 1", "Line 2", "Line X"]
+    hunk = Hunk(
+        2, 2, 2, 2, [(" ", "Line 2"), ("-", "Line 3"), ("+", "Line 3 modified")]
+    )
+    with pytest.raises(
+        DiffApplyError, match="Removing line that doesn't exactly match"
+    ):
+        apply_hunk(result_lines, hunk, 0)
+
+
+def test_apply_hunk_context_mismatch():
+    result_lines = ["Line 1", "Line 3"]
+    hunk = Hunk(2, 2, 2, 2, [(" ", "Line 1"), ("+", "Line 2"), (" ", "Line 4")])
+    with pytest.raises(DiffApplyError, match="Context line doesn't exactly match"):
+        apply_hunk(result_lines, hunk, 0)
+
+
+def test_apply_hunk_invalid_prefix():
+    result_lines = ["Line 1", "Line 2", "Line 3"]
+    hunk = Hunk(
+        2, 2, 2, 2, [(" ", "Line 2"), ("*", "Line 3"), ("+", "Line 3 modified")]
+    )
+    with pytest.raises(DiffApplyError, match="Invalid line prefix"):
+        apply_hunk(result_lines, hunk, 0)
+
+
+def test_apply_hunk_end_of_file():
+    result_lines = ["Line 1", "Line 2"]
+    hunk = Hunk(
+        2, 2, 2, 3, [(" ", "Line 2"), ("-", "Line 3"), ("+", "Line 3 modified")]
+    )
+    with pytest.raises(
+        DiffApplyError, match="Found context or deletions after end of file"
+    ):
+        apply_hunk(result_lines, hunk, 0)
+
+
+def test_apply_hunk_context_beyond_end_of_file():
+    result_lines = ["Line 1", "Line 3"]
+    hunk = Hunk(
+        2, 2, 2, 3, [(" ", "Line 1"), ("+", "Line 2"), (" ", "Line 3"), (" ", "Line 4")]
+    )
+    with pytest.raises(
+        DiffApplyError, match="Found context or deletions after end of file"
+    ):
+        apply_hunk(result_lines, hunk, 0)
+
+
+def test_apply_hunk_remove_non_existent_line():
+    result_lines = ["Line 1", "Line 2", "Line 4"]
+    hunk = Hunk(
+        2, 2, 2, 3, [(" ", "Line 2"), ("-", "Line 3"), ("+", "Line 3 modified")]
+    )
+    with pytest.raises(
+        DiffApplyError, match="Removing line that doesn't exactly match"
+    ):
+        apply_hunk(result_lines, hunk, 0)
+
+
+def test_apply_hunk_addition_beyond_end_of_file():
+    result_lines = ["Line 1", "Line 2"]
+    hunk = Hunk(
+        2, 2, 2, 3, [(" ", "Line 2"), ("+", "Line 3 modified"), ("+", "Line 4")]
+    )
+    apply_hunk(result_lines, hunk, 0)
+    assert result_lines == ["Line 1", "Line 2", "Line 3 modified", "Line 4"]
+
+
+def test_apply_diff():
+    source = """Line 1
+Line 2
+Line 3
+Line 4"""
+    patch = """@@ -1,4 +1,5 @@
+ Line 1
+-Line 2
++Line 2 modified
++Line 2.5
+ Line 3
+ Line 4"""
+    result = apply_diff(source, patch)
+    expected = """Line 1
+Line 2 modified
+Line 2.5
+Line 3
+Line 4
+"""
+    assert result == expected
+
+
+def test_apply_diff_invalid_patch():
+    source = "Line 1\nLine 2\n"
+    patch = "Invalid patch"
+    with pytest.raises(InvalidDiffError):
+        apply_diff(source, patch)
+
+
+def test_apply_diff_unapplicable_patch():
+    source = "Line 1\nLine 2\n"
+    patch = "@@ -1,2 +1,2 @@\n Line 1\n-Line X\n+Line 2 modified\n"
+    with pytest.raises(DiffApplyError):
+        apply_diff(source, patch)
+
+
+def test_apply_diff_add_to_empty_file():
+    source = ""
+    patch = """\
+@@ -1,0 +1,1 @@
++Line 1
++Line 2
+"""
+    result = apply_diff(source, patch)
+    assert result == "Line 1\nLine 2\n"
diff --git a/metadata-ingestion/tests/unit/test_utilities.py b/metadata-ingestion/tests/unit/utilities/test_utilities.py
similarity index 72%
rename from metadata-ingestion/tests/unit/test_utilities.py
rename to metadata-ingestion/tests/unit/utilities/test_utilities.py
index fc2aa27f70b43..c333ceb136fff 100644
--- a/metadata-ingestion/tests/unit/test_utilities.py
+++ b/metadata-ingestion/tests/unit/utilities/test_utilities.py
@@ -1,7 +1,56 @@
-import doctest
+import re
+from typing import List
 
+import datahub.utilities.logging_manager
+from datahub.sql_parsing.schema_resolver import SchemaResolver
+from datahub.sql_parsing.sqlglot_lineage import sqlglot_lineage
+from datahub.testing.doctest import assert_doctest
 from datahub.utilities.delayed_iter import delayed_iter
-from datahub.utilities.sql_parser import SqlLineageSQLParser
+from datahub.utilities.is_pytest import is_pytest_running
+from datahub.utilities.urns.dataset_urn import DatasetUrn
+
+
+class SqlLineageSQLParser:
+    """
+    It uses `sqlglot_lineage` to extract tables and columns, serving as a replacement for the `sqllineage` implementation, similar to BigQuery.
+    Reference: [BigQuery SQL Lineage Test](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/tests/unit/bigquery/test_bigquery_sql_lineage.py#L8).
+    """
+
+    _MYVIEW_SQL_TABLE_NAME_TOKEN = "__my_view__.__sql_table_name__"
+    _MYVIEW_LOOKER_TOKEN = "my_view.SQL_TABLE_NAME"
+
+    def __init__(self, sql_query: str, platform: str = "bigquery") -> None:
+        # SqlLineageParser lowercarese tablenames and we need to replace Looker specific token which should be uppercased
+        sql_query = re.sub(
+            rf"(\${{{self._MYVIEW_LOOKER_TOKEN}}})",
+            rf"{self._MYVIEW_SQL_TABLE_NAME_TOKEN}",
+            sql_query,
+        )
+        self.sql_query = sql_query
+        self.schema_resolver = SchemaResolver(platform=platform)
+        self.result = sqlglot_lineage(sql_query, self.schema_resolver)
+
+    def get_tables(self) -> List[str]:
+        ans = []
+        for urn in self.result.in_tables:
+            table_ref = DatasetUrn.from_string(urn)
+            ans.append(str(table_ref.name))
+
+        result = [
+            self._MYVIEW_LOOKER_TOKEN if c == self._MYVIEW_SQL_TABLE_NAME_TOKEN else c
+            for c in ans
+        ]
+        # Sort tables to make the list deterministic
+        result.sort()
+
+        return result
+
+    def get_columns(self) -> List[str]:
+        ans = []
+        for col_info in self.result.column_lineage or []:
+            for col_ref in col_info.upstreams:
+                ans.append(col_ref.column)
+        return ans
 
 
 def test_delayed_iter():
@@ -120,7 +169,7 @@ def test_sqllineage_sql_parser_get_columns_with_alias_and_count_star():
 
     columns_list = SqlLineageSQLParser(sql_query).get_columns()
     columns_list.sort()
-    assert columns_list == ["a", "b", "count", "test"]
+    assert columns_list == ["a", "b", "c"]
 
 
 def test_sqllineage_sql_parser_get_columns_with_more_complex_join():
@@ -144,7 +193,7 @@ def test_sqllineage_sql_parser_get_columns_with_more_complex_join():
 
     columns_list = SqlLineageSQLParser(sql_query).get_columns()
     columns_list.sort()
-    assert columns_list == ["bs", "pi", "pt", "pu", "v"]
+    assert columns_list == ["bs", "pi", "tt", "tt", "v"]
 
 
 def test_sqllineage_sql_parser_get_columns_complex_query_with_union():
@@ -197,7 +246,7 @@ def test_sqllineage_sql_parser_get_columns_complex_query_with_union():
 
     columns_list = SqlLineageSQLParser(sql_query).get_columns()
     columns_list.sort()
-    assert columns_list == ["c", "date", "e", "u", "x"]
+    assert columns_list == ["c", "c", "e", "e", "e", "e", "u", "u", "x", "x"]
 
 
 def test_sqllineage_sql_parser_get_tables_from_templated_query():
@@ -238,7 +287,7 @@ def test_sqllineage_sql_parser_with_weird_lookml_query():
     """
     columns_list = SqlLineageSQLParser(sql_query).get_columns()
     columns_list.sort()
-    assert columns_list == ["aliased_platform", "country", "date"]
+    assert columns_list == []
 
 
 def test_sqllineage_sql_parser_tables_from_redash_query():
@@ -275,23 +324,14 @@ def test_sqllineage_sql_parser_tables_with_special_names():
         "hour-table",
         "timestamp-table",
     ]
-    expected_columns = [
-        "column-admin",
-        "column-data",
-        "column-date",
-        "column-hour",
-        "column-timestamp",
-    ]
+    expected_columns: List[str] = []
     assert sorted(SqlLineageSQLParser(sql_query).get_tables()) == expected_tables
     assert sorted(SqlLineageSQLParser(sql_query).get_columns()) == expected_columns
 
 
-def test_logging_name_extraction():
-    import datahub.utilities.logging_manager
+def test_logging_name_extraction() -> None:
+    assert_doctest(datahub.utilities.logging_manager)
+
 
-    assert (
-        doctest.testmod(
-            datahub.utilities.logging_manager, raise_on_error=True
-        ).attempted
-        > 0
-    )
+def test_is_pytest_running() -> None:
+    assert is_pytest_running()
diff --git a/metadata-integration/java/acryl-spark-lineage/README.md b/metadata-integration/java/acryl-spark-lineage/README.md
index 9caa5a6dec65d..97851e90e860e 100644
--- a/metadata-integration/java/acryl-spark-lineage/README.md
+++ b/metadata-integration/java/acryl-spark-lineage/README.md
@@ -125,9 +125,10 @@ information like tokens.
   configurations under `Spark Config`.
 
     ```text
-    spark.extraListeners                datahub.spark.DatahubSparkListener
-    spark.datahub.rest.server           http://localhost:8080
-    spark.datahub.databricks.cluster    cluster-name<any preferred cluster identifier>
+    spark.extraListeners                    datahub.spark.DatahubSparkListener
+    spark.datahub.rest.server               http://localhost:8080
+    spark.datahub.stage_metadata_coalescing true
+    spark.datahub.databricks.cluster        cluster-name<any preferred cluster identifier>
     ```
 
 - Click the **Init Scripts** tab. Set cluster init script as `dbfs:/datahub/init.sh`.
@@ -165,6 +166,7 @@ information like tokens.
 | spark.datahub.rest.server                              |          | http://localhost:8080 | Datahub server url  eg:<http://localhost:8080>                                                                                                                                            |
 | spark.datahub.rest.token                               |          |                       | Authentication token.                                                                                                                                                                     |
 | spark.datahub.rest.disable_ssl_verification            |          | false                 | Disable SSL certificate validation. Caution: Only use this if you know what you are doing!                                                                                                |
+| spark.datahub.rest.disable_chunked_encoding            |          | false                 | Disable Chunked Transfer Encoding. In some environment chunked encoding causes issues. With this config option it can be disabled.                                                        ||
 | spark.datahub.rest.max_retries                         |          | 0                     | Number of times a request retried if failed                                                                                                                                               |
 | spark.datahub.rest.retry_interval                      |          | 10                    | Number of seconds to wait between retries                                                                                                                                                 |
 | spark.datahub.file.filename                            |          |                       | The file where metadata will be written if file emitter is set                                                                                                                            |
@@ -184,7 +186,7 @@ information like tokens.
 | spark.datahub.platform.s3.path_spec_list               |          |                       | List of pathspec per platform                                                                                                                                                             |
 | spark.datahub.metadata.dataset.include_schema_metadata | false    |                       | Emit dataset schema metadata based on the spark execution. It is recommended to get schema information from platform specific DataHub sources  as this is less reliable                   |
 | spark.datahub.flow_name                                |          |                       | If it is set it will be used as the DataFlow name otherwise it uses spark app name as flow_name                                                                                           |
-| spark.datahub.partition_regexp_pattern                 |          |                       | Strip partition part from the path if path end matches with the specified regexp. Example `year=.*/month=.*/day=.*`                                                                       |
+| spark.datahub.file_partition_regexp                    |          |                       | Strip partition part from the path if path end matches with the specified regexp. Example `year=.*/month=.*/day=.*`                                                                       |
 | spark.datahub.tags                                     |          |                       | Comma separated list of tags to attach to the DataFlow                                                                                                                                    |
 | spark.datahub.domains                                  |          |                       | Comma separated list of domain urns to attach to the DataFlow                                                                                                                             |
 | spark.datahub.stage_metadata_coalescing                |          |                       | Normally it coalesces and sends metadata at the onApplicationEnd event which is never called on Databricks or on Glue. You should enable this on Databricks if you want coalesced run.    |
diff --git a/metadata-integration/java/acryl-spark-lineage/build.gradle b/metadata-integration/java/acryl-spark-lineage/build.gradle
index 6620c34021ac4..8816264fbe50f 100644
--- a/metadata-integration/java/acryl-spark-lineage/build.gradle
+++ b/metadata-integration/java/acryl-spark-lineage/build.gradle
@@ -1,12 +1,12 @@
 plugins {
   id("com.palantir.git-version") apply false
 }
-apply plugin: 'java'
-apply plugin: 'com.github.johnrengelman.shadow'
+apply plugin: 'java-library'
+apply plugin: 'com.gradleup.shadow'
 apply plugin: 'signing'
 apply plugin: 'io.codearte.nexus-staging'
 apply plugin: 'maven-publish'
-apply plugin: 'jacoco'
+apply from: '../../../gradle/coverage/java-coverage.gradle'
 apply from: '../versioning.gradle'
 
 jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
@@ -51,8 +51,8 @@ dependencies {
 
   implementation project(':metadata-integration:java:openlineage-converter')
 
-  implementation project(path: ':metadata-integration:java:datahub-client', configuration: 'shadow')
-  implementation project(path: ':metadata-integration:java:openlineage-converter', configuration: 'shadow')
+  implementation project(path: ':metadata-integration:java:datahub-client')
+  implementation project(path: ':metadata-integration:java:openlineage-converter')
 
   //implementation "io.acryl:datahub-client:0.10.2"
   implementation "io.openlineage:openlineage-spark_2.12:$openLineageVersion"
@@ -91,6 +91,8 @@ shadowJar {
   zip64 = true
   archiveClassifier = ''
   mergeServiceFiles()
+  project.configurations.implementation.canBeResolved = true
+  configurations = [project.configurations.implementation]
 
   def exclude_modules = project
       .configurations
@@ -106,6 +108,8 @@ shadowJar {
     exclude(dependency {
       exclude_modules.contains(it.name)
     })
+    exclude(dependency("org.slf4j::"))
+    exclude("org/apache/commons/logging/**")
   }
 
   // preventing java multi-release JAR leakage
@@ -123,53 +127,46 @@ shadowJar {
   relocate 'com.sun.activation', 'io.acryl.shaded.com.sun.activation'
   relocate 'com.sun.codemodel', 'io.acryl.shaded.com.sun.codemodel'
   relocate 'com.sun.mail', 'io.acryl.shaded.com.sun.mail'
-  relocate 'com.fasterxml.jackson', 'datahub.spark2.shaded.jackson'
-  relocate 'org.slf4j', 'datahub.spark2.shaded.org.slf4j'
   //
   relocate 'org.apache.hc', 'io.acryl.shaded.http'
-  relocate 'org.apache.commons.codec', 'datahub.spark2.shaded.o.a.c.codec'
-  relocate 'org.apache.commons.compress', 'datahub.spark2.shaded.o.a.c.compress'
-  relocate 'org.apache.commons.lang3', 'datahub.spark2.shaded.o.a.c.lang3'
+  relocate 'org.apache.commons.codec', 'io.acryl.shaded.org.apache.commons.codec'
+  relocate 'org.apache.commons.compress', 'io.acryl.shaded.org.apache.commons.compress'
+  relocate 'org.apache.commons.lang3', 'io.acryl.shaded.org.apache.commons.lang3'
   relocate 'mozilla', 'datahub.spark2.shaded.mozilla'
-  relocate 'com.typesafe', 'datahub.spark2.shaded.typesafe'
-  relocate 'io.opentracing', 'datahub.spark2.shaded.io.opentracing'
-  relocate 'io.netty', 'datahub.spark2.shaded.io.netty'
-  relocate 'ch.randelshofer', 'datahub.spark2.shaded.ch.randelshofer'
-  relocate 'ch.qos', 'datahub.spark2.shaded.ch.qos'
+  relocate 'com.typesafe', 'io.acryl.shaded.com.typesafe'
+  relocate 'io.opentracing', 'io.acryl.shaded.io.opentracing'
+  relocate 'io.netty', 'io.acryl.shaded.io.netty'
+  relocate 'ch.randelshofer', 'io.acryl.shaded.ch.randelshofer'
+  relocate 'ch.qos', 'io.acryl.shaded.ch.qos'
   relocate 'org.springframework', 'io.acryl.shaded.org.springframework'
   relocate 'com.fasterxml.jackson', 'io.acryl.shaded.jackson'
   relocate 'org.yaml', 'io.acryl.shaded.org.yaml' // Required for shading snakeyaml
   relocate 'net.jcip.annotations', 'io.acryl.shaded.annotations'
   relocate 'javassist', 'io.acryl.shaded.javassist'
   relocate 'edu.umd.cs.findbugs', 'io.acryl.shaded.findbugs'
-  relocate 'org.antlr', 'io.acryl.shaded.org.antlr'
-  relocate 'antlr', 'io.acryl.shaded.antlr'
+  //relocate 'org.antlr', 'io.acryl.shaded.org.antlr'
+  //relocate 'antlr', 'io.acryl.shaded.antlr'
   relocate 'com.google.common', 'io.acryl.shaded.com.google.common'
-  relocate 'org.apache.commons', 'io.acryl.shaded.org.apache.commons'
   relocate 'org.reflections', 'io.acryl.shaded.org.reflections'
   relocate 'st4hidden', 'io.acryl.shaded.st4hidden'
   relocate 'org.stringtemplate', 'io.acryl.shaded.org.stringtemplate'
   relocate 'org.abego.treelayout', 'io.acryl.shaded.treelayout'
-  relocate 'org.slf4j', 'io.acryl.shaded.slf4j'
   relocate 'javax.annotation', 'io.acryl.shaded.javax.annotation'
   relocate 'com.github.benmanes.caffeine', 'io.acryl.shaded.com.github.benmanes.caffeine'
   relocate 'org.checkerframework', 'io.acryl.shaded.org.checkerframework'
   relocate 'com.google.errorprone', 'io.acryl.shaded.com.google.errorprone'
   relocate 'com.sun.jna', 'io.acryl.shaded.com.sun.jna'
+
 }
 
 checkShadowJar {
   dependsOn shadowJar
 }
 
-jacocoTestReport {
-  dependsOn test // tests are required to run before generating the report
-}
 
 test {
   forkEvery = 1
   useJUnit()
-  finalizedBy jacocoTestReport
 }
 
 assemble {
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
index 7a5fdeaeb8e0d..0bcc7db9e8740 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
@@ -82,7 +82,10 @@ private Optional<Emitter> getEmitter() {
             (KafkaDatahubEmitterConfig) datahubConf.getDatahubEmitterConfig();
         try {
           emitter =
-              Optional.of(new KafkaEmitter(datahubKafkaEmitterConfig.getKafkaEmitterConfig()));
+              Optional.of(
+                  new KafkaEmitter(
+                      datahubKafkaEmitterConfig.getKafkaEmitterConfig(),
+                      datahubKafkaEmitterConfig.getMcpTopic()));
         } catch (IOException e) {
           throw new RuntimeException(e);
         }
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
index 52507a682a1f8..b594f6bae954f 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
@@ -120,7 +120,9 @@ public Optional<DatahubEmitterConfig> initializeEmitter(Config sparkConf) {
         boolean disableSslVerification =
             sparkConf.hasPath(SparkConfigParser.DISABLE_SSL_VERIFICATION_KEY)
                 && sparkConf.getBoolean(SparkConfigParser.DISABLE_SSL_VERIFICATION_KEY);
-
+        boolean disableChunkedEncoding =
+            sparkConf.hasPath(SparkConfigParser.REST_DISABLE_CHUNKED_ENCODING)
+                && sparkConf.getBoolean(SparkConfigParser.REST_DISABLE_CHUNKED_ENCODING);
         int retry_interval_in_sec =
             sparkConf.hasPath(SparkConfigParser.RETRY_INTERVAL_IN_SEC)
                 ? sparkConf.getInt(SparkConfigParser.RETRY_INTERVAL_IN_SEC)
@@ -150,6 +152,7 @@ public Optional<DatahubEmitterConfig> initializeEmitter(Config sparkConf) {
                 .disableSslVerification(disableSslVerification)
                 .maxRetries(max_retries)
                 .retryIntervalSec(retry_interval_in_sec)
+                .disableChunkedEncoding(disableChunkedEncoding)
                 .build();
         return Optional.of(new RestDatahubEmitterConfig(restEmitterConf));
       case "kafka":
@@ -188,8 +191,12 @@ public Optional<DatahubEmitterConfig> initializeEmitter(Config sparkConf) {
                   });
           kafkaEmitterConfig.producerConfig(kafkaConfig);
         }
-
-        return Optional.of(new KafkaDatahubEmitterConfig(kafkaEmitterConfig.build()));
+        if (sparkConf.hasPath(SparkConfigParser.KAFKA_MCP_TOPIC)) {
+          String mcpTopic = sparkConf.getString(SparkConfigParser.KAFKA_MCP_TOPIC);
+          return Optional.of(new KafkaDatahubEmitterConfig(kafkaEmitterConfig.build(), mcpTopic));
+        } else {
+          return Optional.of(new KafkaDatahubEmitterConfig(kafkaEmitterConfig.build()));
+        }
       case "file":
         log.info("File Emitter Configuration: File emitter will be used");
         FileEmitterConfig.FileEmitterConfigBuilder fileEmitterConfig = FileEmitterConfig.builder();
@@ -370,7 +377,8 @@ private static void initializeMetrics(OpenLineageConfig openLineageConfig) {
     String disabledFacets;
     if (openLineageConfig.getFacetsConfig() != null
         && openLineageConfig.getFacetsConfig().getDisabledFacets() != null) {
-      disabledFacets = String.join(";", openLineageConfig.getFacetsConfig().getDisabledFacets());
+      disabledFacets =
+          String.join(";", openLineageConfig.getFacetsConfig().getEffectiveDisabledFacets());
     } else {
       disabledFacets = "";
     }
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/KafkaDatahubEmitterConfig.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/KafkaDatahubEmitterConfig.java
index 6ed66dbc9230f..a5f9b59f70846 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/KafkaDatahubEmitterConfig.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/KafkaDatahubEmitterConfig.java
@@ -1,5 +1,6 @@
 package datahub.spark.conf;
 
+import datahub.client.kafka.KafkaEmitter;
 import datahub.client.kafka.KafkaEmitterConfig;
 import lombok.Getter;
 import lombok.Setter;
@@ -11,8 +12,15 @@
 public class KafkaDatahubEmitterConfig implements DatahubEmitterConfig {
   final String type = "kafka";
   KafkaEmitterConfig kafkaEmitterConfig;
+  String mcpTopic;
 
   public KafkaDatahubEmitterConfig(KafkaEmitterConfig kafkaEmitterConfig) {
     this.kafkaEmitterConfig = kafkaEmitterConfig;
+    this.mcpTopic = KafkaEmitter.DEFAULT_MCP_KAFKA_TOPIC;
+  }
+
+  public KafkaDatahubEmitterConfig(KafkaEmitterConfig kafkaEmitterConfig, String mcpTopic) {
+    this.kafkaEmitterConfig = kafkaEmitterConfig;
+    this.mcpTopic = mcpTopic;
   }
 }
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
index 630f10b08b411..3860285083c4b 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
@@ -30,8 +30,11 @@ public class SparkConfigParser {
   public static final String GMS_AUTH_TOKEN = "rest.token";
   public static final String FILE_EMITTER_FILE_NAME = "file.filename";
   public static final String DISABLE_SSL_VERIFICATION_KEY = "rest.disable_ssl_verification";
+  public static final String REST_DISABLE_CHUNKED_ENCODING = "rest.disable_chunked_encoding";
+
   public static final String MAX_RETRIES = "rest.max_retries";
   public static final String RETRY_INTERVAL_IN_SEC = "rest.retry_interval_in_sec";
+  public static final String KAFKA_MCP_TOPIC = "kafka.mcp_topic";
   public static final String KAFKA_EMITTER_BOOTSTRAP = "kafka.bootstrap";
   public static final String KAFKA_EMITTER_SCHEMA_REGISTRY_URL = "kafka.schema_registry_url";
   public static final String KAFKA_EMITTER_SCHEMA_REGISTRY_CONFIG = "kafka.schema_registry_config";
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java
index d46d741d155b8..5f87df2a65d6c 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java
@@ -5,14 +5,13 @@
 
 package io.openlineage.spark.agent.util;
 
-import static io.openlineage.spark.agent.lifecycle.ExecutionContext.CAMEL_TO_SNAKE_CASE;
-
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import datahub.spark.conf.SparkLineageConf;
 import io.datahubproject.openlineage.dataset.HdfsPathDataset;
 import io.openlineage.client.OpenLineage;
 import io.openlineage.spark.agent.Versions;
+import io.openlineage.spark.api.naming.NameNormalizer;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -21,7 +20,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.UUID;
@@ -186,7 +184,7 @@ public static OpenLineage.ParentRunFacet parentRunFacet(
         .run(new OpenLineage.ParentRunFacetRunBuilder().runId(parentRunId).build())
         .job(
             new OpenLineage.ParentRunFacetJobBuilder()
-                .name(parentJob.replaceAll(CAMEL_TO_SNAKE_CASE, "_$1").toLowerCase(Locale.ROOT))
+                .name(NameNormalizer.normalize(parentJob))
                 .namespace(parentJobNamespace)
                 .build())
         .build();
@@ -287,8 +285,6 @@ public static boolean safeIsDefinedAt(PartialFunction pfn, Object x) {
    * @param pfn
    * @param x
    * @return
-   * @param <T>
-   * @param <D>
    */
   public static <T, D> List<T> safeApply(PartialFunction<D, List<T>> pfn, D x) {
     try {
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java
index 62005bf15f850..6ef7403362a90 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java
@@ -7,6 +7,7 @@
 
 import java.util.Arrays;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Stream;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.reflect.FieldUtils;
@@ -18,6 +19,7 @@
 import org.apache.spark.rdd.MapPartitionsRDD;
 import org.apache.spark.rdd.ParallelCollectionRDD;
 import org.apache.spark.rdd.RDD;
+import org.apache.spark.sql.execution.datasources.FilePartition;
 import org.apache.spark.sql.execution.datasources.FileScanRDD;
 import scala.Tuple2;
 import scala.collection.immutable.Seq;
@@ -90,7 +92,7 @@ public boolean isDefinedAt(Object rdd) {
     @SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
     public Stream<Path> extract(FileScanRDD rdd) {
       return ScalaConversionUtils.fromSeq(rdd.filePartitions()).stream()
-          .flatMap(fp -> Arrays.stream(fp.files()))
+          .flatMap((FilePartition fp) -> Arrays.stream(fp.files()))
           .map(
               f -> {
                 if ("3.4".compareTo(package$.MODULE$.SPARK_VERSION()) <= 0) {
@@ -115,11 +117,15 @@ public boolean isDefinedAt(Object rdd) {
 
     @Override
     public Stream<Path> extract(ParallelCollectionRDD rdd) {
+      int SEQ_LIMIT = 1000;
+      AtomicBoolean loggingDone = new AtomicBoolean(false);
       try {
         Object data = FieldUtils.readField(rdd, "data", true);
         log.debug("ParallelCollectionRDD data: {}", data);
-        if (data instanceof Seq) {
-          return ScalaConversionUtils.fromSeq((Seq) data).stream()
+        if ((data instanceof Seq) && ((Seq) data).head() instanceof Tuple2) {
+          // exit if the first element is invalid
+          Seq data_slice = (Seq) ((Seq) data).slice(0, SEQ_LIMIT);
+          return ScalaConversionUtils.fromSeq(data_slice).stream()
               .map(
                   el -> {
                     Path path = null;
@@ -127,9 +133,9 @@ public Stream<Path> extract(ParallelCollectionRDD rdd) {
                       // we're able to extract path
                       path = parentOf(((Tuple2) el)._1.toString());
                       log.debug("Found input {}", path);
-                    } else {
-                      // Change to debug to silence error
-                      log.debug("unable to extract Path from {}", el.getClass().getCanonicalName());
+                    } else if (!loggingDone.get()) {
+                      log.warn("unable to extract Path from {}", el.getClass().getCanonicalName());
+                      loggingDone.set(true);
                     }
                     return path;
                   })
diff --git a/metadata-integration/java/as-a-library.md b/metadata-integration/java/as-a-library.md
index dea0a2c94545e..907b331b8bd96 100644
--- a/metadata-integration/java/as-a-library.md
+++ b/metadata-integration/java/as-a-library.md
@@ -67,8 +67,8 @@ MetadataChangeProposalWrapper mcpw = MetadataChangeProposalWrapper.builder()
         .aspect(new DatasetProperties().setDescription("This is the canonical User profile dataset"))
         .build();
 
-// Blocking call using future
-Future<MetadataWriteResponse> requestFuture = emitter.emit(mcpw, null).get();
+// Blocking call using Future.get()
+MetadataWriteResponse requestFuture = emitter.emit(mcpw, null).get();
 
 // Non-blocking using callback
 emitter.emit(mcpw, new Callback() {
diff --git a/metadata-integration/java/custom-plugin-lib/build.gradle b/metadata-integration/java/custom-plugin-lib/build.gradle
index 305dbe9578fa0..11933a947487f 100644
--- a/metadata-integration/java/custom-plugin-lib/build.gradle
+++ b/metadata-integration/java/custom-plugin-lib/build.gradle
@@ -1,6 +1,6 @@
 plugins {
   id 'java-library'
-  id 'com.github.johnrengelman.shadow'
+  id 'com.gradleup.shadow'
   id 'signing'
   id 'io.codearte.nexus-staging'
   id 'maven-publish'
diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle
index d9087347e1b5c..42861cf235b56 100644
--- a/metadata-integration/java/datahub-client/build.gradle
+++ b/metadata-integration/java/datahub-client/build.gradle
@@ -1,22 +1,29 @@
 plugins {
   id("com.palantir.git-version") apply false
-  id 'java'
-  id 'com.github.johnrengelman.shadow'
-  id 'jacoco'
+  id 'java-library'
+  id 'com.gradleup.shadow'
   id 'signing'
   id 'io.codearte.nexus-staging'
   id 'maven-publish'
 }
 
+apply from: "../../../gradle/coverage/java-coverage.gradle"
 apply from: "../versioning.gradle"
 import org.apache.tools.ant.filters.ReplaceTokens
 
 
-jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
+jar {
+  if (project.hasProperty('archiveAppendix')) {
+    archiveAppendix.set(project.archiveAppendix)
+  }
+  archiveClassifier = "lib"
+}
 
 dependencies {
-  implementation project(':entity-registry')
-  implementation project(':metadata-integration:java:datahub-event')
+  api project(':entity-registry')
+  api project(':metadata-integration:java:datahub-event')
+  implementation project(':metadata-integration:java:datahub-schematron:lib')
+
   implementation(externalDependency.kafkaAvroSerializer) {
     exclude group: "org.apache.avro"
   }
@@ -33,7 +40,7 @@ dependencies {
   implementation externalDependency.jacksonDataBind
   runtimeOnly externalDependency.jna
 
-  implementation externalDependency.slf4jApi
+  api externalDependency.slf4jApi
   compileOnly externalDependency.lombok
   annotationProcessor externalDependency.lombok
   // VisibleForTesting
@@ -56,28 +63,62 @@ task copyAvroSchemas {
 
 compileJava.dependsOn copyAvroSchemas
 
-jacocoTestReport {
-  dependsOn test // tests are required to run before generating the report
+
+// Add Python environment validation task
+task validatePythonEnv(dependsOn: [":metadata-ingestion:installDev"]) {
+    doFirst {
+        def venvPath = System.getProperty('python.venv.path', '../../../metadata-ingestion/venv')
+        def isWindows = System.getProperty('os.name').toLowerCase().contains('windows')
+        def pythonExe = isWindows ? "${venvPath}/Scripts/python.exe" : "${venvPath}/bin/python"
+
+        def result = exec {
+            commandLine pythonExe, "-c", "import sys; print(sys.executable)"
+            ignoreExitValue = true
+            standardOutput = new ByteArrayOutputStream()
+            errorOutput = new ByteArrayOutputStream()
+        }
+
+        if (result.exitValue != 0) {
+            throw new GradleException("Python virtual environment not properly set up at ${venvPath}")
+        }
+    }
 }
 
 test {
   // to avoid simultaneous executions of tests when complete build is run
   mustRunAfter(":metadata-io:test")
   useJUnit()
+  // Add Python environment configuration
+  dependsOn validatePythonEnv
+  dependsOn tasks.getByPath(":metadata-ingestion:installDev")
+  systemProperty 'python.venv.path', System.getProperty('python.venv.path', '../../../metadata-ingestion/venv')
   finalizedBy jacocoTestReport
 }
 
+// no submodule depends on datahub-schematron:cli
+// and tests there are the ones checking python-java compatibility
+test.dependsOn tasks.getByPath(":metadata-integration:java:datahub-schematron:cli:test")
+test.dependsOn tasks.getByPath(":metadata-integration:java:datahub-schematron:lib:test")
+
 task checkShadowJar(type: Exec) {
   commandLine 'sh', '-c', 'scripts/check_jar.sh'
 }
 
 shadowJar {
   zip64 = true
+  if (project.hasProperty('archiveAppendix')) {
+    archiveAppendix.set(project.archiveAppendix)
+  }
   archiveClassifier = ''
   // preventing java multi-release JAR leakage
   // https://github.com/johnrengelman/shadow/issues/729
   exclude('module-info.class', 'META-INF/versions/**',
       '**/LICENSE', '**/LICENSE*.txt', '**/NOTICE', '**/NOTICE.txt', 'licenses/**', 'log4j2.*', 'log4j.*')
+  dependencies {
+    exclude(dependency("org.slf4j::"))
+    exclude(dependency("antlr::"))
+    exclude("org/apache/commons/logging/**")
+  }
   mergeServiceFiles()
   // we relocate namespaces manually, because we want to know exactly which libs we are exposing and why
   // we can move to automatic relocation using ConfigureShadowRelocation after we get to a good place on these first
@@ -88,21 +129,25 @@ shadowJar {
   relocate 'javassist', 'datahub.shaded.javassist'
   relocate 'edu.umd.cs.findbugs', 'datahub.shaded.findbugs'
   relocate 'org.antlr', 'datahub.shaded.org.antlr'
-  relocate 'antlr', 'datahub.shaded.antlr'
+  //relocate 'antlr', 'datahub.shaded.antlr'
   relocate 'com.google.common', 'datahub.shaded.com.google.common'
-  relocate 'org.apache.commons', 'datahub.shaded.org.apache.commons'
+  relocate 'org.apache.commons.codec', 'datahub.shaded.org.apache.commons.codec'
+  relocate 'org.apache.commons.compress', 'datahub.shaded.org.apache.commons.compress'
+  relocate 'org.apache.commons.lang3', 'datahub.shaded.org.apache.commons.lang3'
+  relocate 'org.apache.commons.lang', 'datahub.shaded.org.apache.commons.lang'
+  relocate 'org.apache.commons.cli', 'datahub.shaded.org.apache.commons.cli'
+  relocate 'org.apache.commons.text', 'datahub.shaded.org.apache.commons.text'
+  relocate 'org.apache.commons.io', 'datahub.shaded.org.apache.commons.io'
   relocate 'org.apache.maven', 'datahub.shaded.org.apache.maven'
   relocate 'org.reflections', 'datahub.shaded.org.reflections'
   relocate 'st4hidden', 'datahub.shaded.st4hidden'
   relocate 'org.stringtemplate', 'datahub.shaded.org.stringtemplate'
   relocate 'org.abego.treelayout', 'datahub.shaded.treelayout'
-  relocate 'org.slf4j', 'datahub.shaded.slf4j'
   relocate 'javax.annotation', 'datahub.shaded.javax.annotation'
   relocate 'com.github.benmanes.caffeine', 'datahub.shaded.com.github.benmanes.caffeine'
   relocate 'org.checkerframework', 'datahub.shaded.org.checkerframework'
   relocate 'com.google.errorprone', 'datahub.shaded.com.google.errorprone'
   // Below jars added for kafka emitter only
-  relocate 'org.apache.avro', 'datahub.shaded.org.apache.avro'
   relocate 'com.thoughtworks.paranamer', 'datahub.shaded.com.thoughtworks.paranamer'
   relocate 'org.xerial.snappy', 'datahub.shaded.org.xerial.snappy'
   relocate 'org.apache.kafka', 'datahub.shaded.org.apache.kafka'
@@ -156,7 +201,7 @@ publishing {
       pom {
         name = 'Datahub Client'
         group = 'io.acryl'
-        artifactId = 'datahub-client'
+        artifactId = 'datahub-client' + (project.hasProperty('archiveAppendix') ? "-${archiveAppendix}" : '')
         description = 'DataHub Java client for metadata integration'
         url = 'https://datahubproject.io'
         artifacts = [shadowJar, javadocJar, sourcesJar]
diff --git a/metadata-integration/java/datahub-client/scripts/check_jar.sh b/metadata-integration/java/datahub-client/scripts/check_jar.sh
index 10299ec714d16..e451a7dd2a009 100755
--- a/metadata-integration/java/datahub-client/scripts/check_jar.sh
+++ b/metadata-integration/java/datahub-client/scripts/check_jar.sh
@@ -40,7 +40,10 @@ jar -tvf $jarFile |\
       grep -v "mozilla"  |\
       grep -v "VersionInfo.java" |\
       grep -v "mime.types" |\
-      grep -v "com/ibm/.*"
+      grep -v "com/ibm/.*" |\
+      grep -v "org/apache/avro" |\
+      grep -v "org/apache"
+
 
 
 if [ $? -ne 0 ]; then
diff --git a/metadata-integration/java/datahub-client/scripts/container_key_guid_generator.py b/metadata-integration/java/datahub-client/scripts/container_key_guid_generator.py
new file mode 100644
index 0000000000000..9fc18a85426bd
--- /dev/null
+++ b/metadata-integration/java/datahub-client/scripts/container_key_guid_generator.py
@@ -0,0 +1,122 @@
+import click
+from typing import Dict, Any
+import json
+from dataclasses import dataclass
+from abc import ABC, abstractmethod
+from datahub.emitter.mcp_builder import DatabaseKey, SchemaKey
+
+
+class URNGenerator(ABC):
+    @abstractmethod
+    def generate(self, args: Dict[str, Any]) -> str:
+        pass
+
+
+class DatabaseURNGenerator(URNGenerator):
+    def generate(self, args: Dict[str, Any]) -> str:
+        required_fields = ["platform", "database"]
+        for field in required_fields:
+            if field not in args:
+                raise ValueError(f"Missing required field: {field}")
+
+        all_fields = required_fields + ["instance"]
+        for arg in args:
+            if arg not in all_fields:
+                raise ValueError(f"Invalid field: {arg}")
+
+        database_key = DatabaseKey(
+            platform=args["platform"],
+            instance=args.get("instance"),
+            database=args["database"],
+        )
+        return database_key.as_urn()
+
+
+class SchemaURNGenerator(URNGenerator):
+    def generate(self, args: Dict[str, Any]) -> str:
+        required_fields = ["platform", "database", "schema"]
+        all_fields = required_fields + ["instance", "env"]
+        for field in required_fields:
+            if field not in args:
+                raise ValueError(f"Missing required field: {field}")
+
+        for arg in args:
+            if arg not in all_fields:
+                raise ValueError(f"Invalid field: {arg}")
+
+        schema_key = SchemaKey(
+            platform=args["platform"],
+            instance=args.get("instance"),
+            env=args.get("env"),
+            database=args["database"],
+            schema=args["schema"],
+        )
+        return schema_key.as_urn()
+
+
+URN_GENERATORS = {
+    "database": DatabaseURNGenerator(),
+    "schema": SchemaURNGenerator(),
+}
+
+
+def validate_key_value(ctx, param, value):
+    if not value:
+        return {}
+
+    result = {}
+    for item in value:
+        try:
+            key, val = item.split("=", 1)
+            result[key.strip()] = val.strip()
+        except ValueError:
+            raise click.BadParameter(
+                f"Invalid key-value pair: {item}. Format should be key=value"
+            )
+    return result
+
+
+@click.command()
+@click.option(
+    "--container-type",
+    type=click.Choice(["database", "schema"]),
+    required=True,
+    help="The type of container to generate a URN for",
+)
+@click.option(
+    "--param",
+    "-p",
+    multiple=True,
+    callback=validate_key_value,
+    help="Parameters in key=value format. Can be used multiple times.",
+)
+@click.option(
+    "--output-format",
+    type=click.Choice(["text", "json"]),
+    default="text",
+    help="Output format for the URN",
+)
+def generate_urn(container_type: str, param: Dict[str, str], output_format: str):
+    """Generate URNs for different types of containers.
+
+    Example usage:
+    ./container_urn_generator.py --container-type database -p platform=test-platform -p instance=DEV -p database=test-database
+    """
+    try:
+        generator = URN_GENERATORS[container_type]
+        urn = generator.generate(param)
+
+        if output_format == "json":
+            result = {"urn": urn, "container_type": container_type, "parameters": param}
+            click.echo(json.dumps(result, indent=2))
+        else:
+            click.echo(urn)
+
+    except KeyError as e:
+        raise click.UsageError(f"Unknown container type: {container_type}")
+    except ValueError as e:
+        raise click.UsageError(str(e))
+
+
+if __name__ == "__main__":
+    generate_urn()
diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
index 0d0341562e7dd..e8ceeab696321 100644
--- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
+++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
@@ -9,7 +9,7 @@
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
 
-class AvroSerializer {
+public class AvroSerializer {
 
   private final Schema _recordSchema;
   private final Schema _genericAspectSchema;
diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java
index d00dc09669045..777d2d5f301d7 100644
--- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java
+++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java
@@ -31,14 +31,25 @@ public class KafkaEmitter implements Emitter {
   private final Properties kafkaConfigProperties;
   private AvroSerializer _avroSerializer;
   private static final int ADMIN_CLIENT_TIMEOUT_MS = 5000;
+  private final String mcpKafkaTopic;
 
   /**
    * The default constructor
    *
-   * @param config
-   * @throws IOException
+   * @param config KafkaEmitterConfig
+   * @throws IOException when Avro Serialization fails
    */
   public KafkaEmitter(KafkaEmitterConfig config) throws IOException {
+    this(config, DEFAULT_MCP_KAFKA_TOPIC);
+  }
+
+  /**
+   * Constructor that takes in KafkaEmitterConfig and mcp Kafka Topic Name
+   *
+   * @param config KafkaEmitterConfig
+   * @throws IOException when Avro Serialization fails
+   */
+  public KafkaEmitter(KafkaEmitterConfig config, String mcpKafkaTopic) throws IOException {
     this.config = config;
     kafkaConfigProperties = new Properties();
     kafkaConfigProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.config.getBootstrap());
@@ -54,6 +65,7 @@ public KafkaEmitter(KafkaEmitterConfig config) throws IOException {
 
     producer = new KafkaProducer<>(kafkaConfigProperties);
     _avroSerializer = new AvroSerializer();
+    this.mcpKafkaTopic = mcpKafkaTopic;
   }
 
   @Override
@@ -73,8 +85,7 @@ public Future<MetadataWriteResponse> emit(MetadataChangeProposal mcp, Callback d
       throws IOException {
     GenericRecord genricRecord = _avroSerializer.serialize(mcp);
     ProducerRecord<Object, Object> record =
-        new ProducerRecord<>(
-            KafkaEmitter.DEFAULT_MCP_KAFKA_TOPIC, mcp.getEntityUrn().toString(), genricRecord);
+        new ProducerRecord<>(this.mcpKafkaTopic, mcp.getEntityUrn().toString(), genricRecord);
     org.apache.kafka.clients.producer.Callback callback =
         new org.apache.kafka.clients.producer.Callback() {
 
diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java
index 71a4b93baf48f..50c0277c98b03 100644
--- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java
+++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java
@@ -48,7 +48,6 @@ public boolean retryRequest(
 
   @Override
   public boolean retryRequest(HttpResponse response, int execCount, HttpContext context) {
-    log.warn("Retrying request due to error: {}", response);
     return super.retryRequest(response, execCount, context);
   }
 }
diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java
index e1017372be124..d70c5baf10879 100644
--- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java
+++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java
@@ -1,6 +1,7 @@
 package datahub.client.rest;
 
 import static com.linkedin.metadata.Constants.*;
+import static org.apache.hc.core5.http.HttpHeaders.*;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.StreamReadConstraints;
@@ -18,6 +19,7 @@
 import datahub.event.UpsertAspectRequest;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.security.KeyManagementException;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
@@ -26,6 +28,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import javax.annotation.concurrent.ThreadSafe;
@@ -97,17 +100,20 @@ public RestEmitter(RestEmitterConfig config) {
     this.config = config;
     HttpAsyncClientBuilder httpClientBuilder = this.config.getAsyncHttpClientBuilder();
     httpClientBuilder.setRetryStrategy(new DatahubHttpRequestRetryStrategy());
-
-    // Override httpClient settings with RestEmitter configs if present
-    if (config.getTimeoutSec() != null) {
-      httpClientBuilder.setDefaultRequestConfig(
-          RequestConfig.custom()
-              .setConnectionRequestTimeout(
-                  config.getTimeoutSec() * 1000, java.util.concurrent.TimeUnit.MILLISECONDS)
-              .setResponseTimeout(
-                  config.getTimeoutSec() * 1000, java.util.concurrent.TimeUnit.MILLISECONDS)
-              .build());
+    if ((config.getTimeoutSec() != null) || (config.isDisableChunkedEncoding())) {
+      RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
+      // Override httpClient settings with RestEmitter configs if present
+      if (config.getTimeoutSec() != null) {
+        requestConfigBuilder
+            .setConnectionRequestTimeout(config.getTimeoutSec() * 1000, TimeUnit.MILLISECONDS)
+            .setResponseTimeout(config.getTimeoutSec() * 1000, TimeUnit.MILLISECONDS);
+      }
+      if (config.isDisableChunkedEncoding()) {
+        requestConfigBuilder.setContentCompressionEnabled(false);
+      }
+      httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());
     }
+
     PoolingAsyncClientConnectionManagerBuilder poolingAsyncClientConnectionManagerBuilder =
         PoolingAsyncClientConnectionManagerBuilder.create();
 
@@ -223,8 +229,13 @@ private Future<MetadataWriteResponse> postGeneric(
     if (this.config.getToken() != null) {
       simpleRequestBuilder.setHeader("Authorization", "Bearer " + this.config.getToken());
     }
+    if (this.config.isDisableChunkedEncoding()) {
+      byte[] payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8);
+      simpleRequestBuilder.setBody(payloadBytes, ContentType.APPLICATION_JSON);
+    } else {
+      simpleRequestBuilder.setBody(payloadJson, ContentType.APPLICATION_JSON);
+    }
 
-    simpleRequestBuilder.setBody(payloadJson, ContentType.APPLICATION_JSON);
     AtomicReference<MetadataWriteResponse> responseAtomicReference = new AtomicReference<>();
     CountDownLatch responseLatch = new CountDownLatch(1);
     FutureCallback<SimpleHttpResponse> httpCallback =
diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java
index e28ad4ed660f0..55c11aab0ebf3 100644
--- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java
+++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java
@@ -30,6 +30,8 @@ public class RestEmitterConfig {
   Integer timeoutSec;
   @Builder.Default boolean disableSslVerification = false;
 
+  @Builder.Default boolean disableChunkedEncoding = false;
+
   @Builder.Default int maxRetries = 0;
 
   @Builder.Default int retryIntervalSec = 10;
diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/ContainerKey.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/ContainerKey.java
new file mode 100644
index 0000000000000..5bc6c829dcaa1
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/ContainerKey.java
@@ -0,0 +1,42 @@
+package io.datahubproject.models.util;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.linkedin.common.urn.Urn;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@EqualsAndHashCode(callSuper = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public abstract class ContainerKey extends DataHubKey {
+  private String platform;
+  private String instance;
+
+  private static final String URN_PREFIX = "urn:li:container:";
+  private static final String URN_ENTITY = "container";
+  private static final String PLATFORM_MAP_FIELD = "platform";
+  private static final String INSTANCE_MAP_FIELD = "instance";
+
+  @Override
+  public Map<String, String> guidDict() {
+
+    Map<String, String> bag = new HashMap<>();
+    if (platform != null) bag.put(PLATFORM_MAP_FIELD, platform);
+    if (instance != null) bag.put(INSTANCE_MAP_FIELD, instance);
+
+    return bag;
+  }
+
+  public String asUrnString() {
+    String guid = guid();
+    return URN_PREFIX + guid;
+  }
+
+  public Urn asUrn() {
+    return Urn.createFromTuple(URN_ENTITY, guid());
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubGuidGenerator.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubGuidGenerator.java
new file mode 100644
index 0000000000000..ea67b7fab1781
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubGuidGenerator.java
@@ -0,0 +1,41 @@
+package io.datahubproject.models.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.security.MessageDigest;
+import java.util.Map;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DataHubGuidGenerator {
+  private static final ObjectMapper objectMapper = new ObjectMapper();
+
+  @SneakyThrows
+  public static String dataHubGuid(Map<String, String> obj) {
+    // Configure ObjectMapper for consistent serialization
+    objectMapper.configure(
+        com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true);
+
+    // Convert map to JSON string with sorted keys
+    String jsonKey = objectMapper.writeValueAsString(obj);
+
+    // Generate MD5 hash
+    MessageDigest md = MessageDigest.getInstance("MD5");
+    byte[] hashBytes = md.digest(jsonKey.getBytes());
+
+    // Convert byte array to hexadecimal string
+    StringBuilder hexString = new StringBuilder();
+    for (byte hashByte : hashBytes) {
+      String hex = Integer.toHexString(0xff & hashByte);
+      if (hex.length() == 1) {
+        hexString.append('0');
+      }
+      hexString.append(hex);
+    }
+
+    if (log.isDebugEnabled()) {
+      log.debug("DataHub Guid for {} is : {}", jsonKey, hexString);
+    }
+    return hexString.toString();
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubKey.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubKey.java
new file mode 100644
index 0000000000000..a047dd8840935
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubKey.java
@@ -0,0 +1,32 @@
+package io.datahubproject.models.util;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Map;
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public abstract class DataHubKey {
+  // Static ObjectMapper instance since it's thread-safe and expensive to create
+  protected static final ObjectMapper MAPPER = new ObjectMapper();
+  // Static TypeReference instance since it doesn't change
+  private static final TypeReference<Map<String, String>> MAP_TYPE_REFERENCE =
+      new TypeReference<Map<String, String>>() {};
+
+  static {
+    MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+  }
+
+  public Map<String, String> guidDict() {
+    return MAPPER.convertValue(this, MAP_TYPE_REFERENCE);
+  }
+
+  public String guid() {
+    Map<String, String> bag = guidDict();
+    return DataHubGuidGenerator.dataHubGuid(bag);
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DatabaseKey.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DatabaseKey.java
new file mode 100644
index 0000000000000..87a79ea2e7440
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DatabaseKey.java
@@ -0,0 +1,30 @@
+package io.datahubproject.models.util;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.Map;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@EqualsAndHashCode(callSuper = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class DatabaseKey extends ContainerKey {
+  private String database;
+
+  private static final String DATABASE_MAP_FIELD = "database";
+
+  @Override
+  public Map<String, String> guidDict() {
+    // Get the parent's GUID dictionary first
+    Map<String, String> bag = super.guidDict();
+
+    // Add the database field if it's not null
+    if (database != null) {
+      bag.put(DATABASE_MAP_FIELD, database);
+    }
+
+    return bag;
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/FieldPath.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/FieldPath.java
new file mode 100644
index 0000000000000..ab8c11ae9b67d
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/FieldPath.java
@@ -0,0 +1,88 @@
+package io.datahubproject.models.util;
+
+import lombok.NonNull;
+
+public class FieldPath {
+  private final String rawFieldPath;
+  private final String[] segments;
+  private final String version;
+  private final String simplePath;
+
+  public FieldPath(@NonNull String rawFieldPath) {
+    this.rawFieldPath = rawFieldPath;
+    this.segments = rawFieldPath.split("\\.");
+    this.version = computeVersion();
+    this.simplePath = computeSimplePath();
+  }
+
+  public boolean isTopLevel() {
+    return depth() == 1;
+  }
+
+  /**
+   * Returns the logical depth of the field path, ignoring type and version metadata Example:
+   * "[version=2.0][type=Foo].address.[type=union].[type=string].street" -> depth = 2
+   *
+   * @return The logical depth of the field path
+   */
+  public int depth() {
+    String[] segments = simplePath().split("\\.");
+    return segments.length;
+  }
+
+  public String leafFieldName() {
+    return segments[segments.length - 1];
+  }
+
+  /**
+   * Extracts the version number from the field path. Example: rawFieldPath = "a.b.c" -> version() =
+   * "1" rawFieldPath = "[version=2].a.b.c" -> version() = "2" rawFieldPath = "[version=20].a.b.c"
+   * -> version() = "20"
+   *
+   * @return String representing the version number
+   */
+  private String computeVersion() {
+    if (rawFieldPath == null) {
+      return "1";
+    }
+
+    // Check for version pattern [version=X] where X can be any non-bracket characters
+    java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\[version=([^\\]]+)\\]");
+    java.util.regex.Matcher matcher = pattern.matcher(rawFieldPath);
+
+    if (matcher.find()) {
+      return matcher.group(1);
+    }
+
+    return "1";
+  }
+
+  public String version() {
+    return version;
+  }
+
+  /**
+   * Returns the simplified path without version, type, or other metadata Example:
+   * "[version=2.0][type=Foo].address.[type=union].[type=string].street" -> "address.street"
+   *
+   * @return The simplified field path
+   */
+  private String computeSimplePath() {
+    if (rawFieldPath == null) {
+      return "";
+    }
+    // Remove all metadata blocks [xxx=yyy]
+    String simplified = rawFieldPath.replaceAll("\\[.*?\\]", "");
+    // Replace all "double dots" with a single dot
+    simplified = simplified.replaceAll("\\.+", ".");
+    // Replace all leading and trailing dots
+    simplified = simplified.replaceAll("^\\.|\\.$", "");
+    // Remove any trailing metadata blocks without dots
+    simplified = simplified.replaceAll("\\[.*?\\]", "");
+    return simplified;
+  }
+
+  public String simplePath() {
+    return simplePath;
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/SchemaKey.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/SchemaKey.java
new file mode 100644
index 0000000000000..800ee43614749
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/SchemaKey.java
@@ -0,0 +1,30 @@
+package io.datahubproject.models.util;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.Map;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@EqualsAndHashCode(callSuper = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SchemaKey extends DatabaseKey {
+  private String schema;
+
+  private static final String SCHEMA_MAP_FIELD = "schema";
+
+  @Override
+  public Map<String, String> guidDict() {
+    // Get the parent's GUID dictionary first
+    Map<String, String> bag = super.guidDict();
+
+    // Add the database field if it's not null
+    if (schema != null) {
+      bag.put(SCHEMA_MAP_FIELD, schema);
+    }
+
+    return bag;
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DataHubGuidGeneratorTest.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DataHubGuidGeneratorTest.java
new file mode 100644
index 0000000000000..8ee65d51de92d
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DataHubGuidGeneratorTest.java
@@ -0,0 +1,61 @@
+package io.datahubproject.models.util;
+
+import static org.junit.Assert.*;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Test;
+
+public class DataHubGuidGeneratorTest {
+
+  @Test
+  public void testGuidGeneration() throws NoSuchAlgorithmException, JsonProcessingException {
+    // Test data
+    Map<String, String> obj = new HashMap<>();
+    obj.put("container", "test-container");
+
+    // Generate GUID
+    String guid = DataHubGuidGenerator.dataHubGuid(obj);
+
+    // Assert
+    assertEquals("4d90f727b9d10ba7cea297dc8b427985", guid);
+  }
+
+  @Test
+  public void testContainerUrnGeneration() {
+    // Test data
+    DatabaseKey databaseKey =
+        DatabaseKey.builder()
+            .platform("test-platform")
+            .instance("DEV")
+            .database("test-database")
+            .build();
+
+    System.out.println(databaseKey.guidDict());
+
+    // Generate URN
+    String urn = databaseKey.asUrnString();
+    // With instance
+    String expectedUrn = "urn:li:container:e40f103ea7c6def4f4b24cd858d5e412";
+
+    // Assert
+    assertEquals(expectedUrn, urn);
+  }
+
+  @Test
+  public void testContainerUrnGenerationNoInstance() {
+    // Test data
+    ContainerKey containerKey =
+        DatabaseKey.builder().platform("test-platform").database("test-database").build();
+
+    // Generate URN
+    String urn = containerKey.asUrnString();
+    // Without instance
+    String expectedUrn = "urn:li:container:1929d86c0a92e2d3bb9ba193c8c2b66f";
+
+    // Assert
+    assertEquals(expectedUrn, urn);
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DatabaseKeyTest.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DatabaseKeyTest.java
new file mode 100644
index 0000000000000..08c6e9ce33a68
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DatabaseKeyTest.java
@@ -0,0 +1,51 @@
+package io.datahubproject.models.util;
+
+import static org.junit.Assert.*;
+
+import java.util.Map;
+import org.junit.Test;
+
+public class DatabaseKeyTest {
+  @Test
+  public void testDatabaseUrnGeneration() {
+    // Test data
+    DatabaseKey databaseKey =
+        DatabaseKey.builder()
+            .platform("test-platform")
+            .instance("DEV")
+            .database("test-database")
+            .build();
+
+    System.out.println(databaseKey.guidDict());
+
+    // Generate URN
+    String urn = databaseKey.asUrnString();
+    // With instance
+    //    "urn:li:container:e40f103ea7c6def4f4b24cd858d5e412";
+    String expectedUrn =
+        TestHelper.generateContainerKeyGuid(
+            "database",
+            Map.of("platform", "test-platform", "instance", "DEV", "database", "test-database"));
+
+    // Assert
+    assertEquals(expectedUrn, urn);
+  }
+
+  @Test
+  public void testDatabaseUrnGenerationNoInstance() {
+    // Test data
+    ContainerKey containerKey =
+        DatabaseKey.builder().platform("test-platform").database("test-database").build();
+
+    // Generate URN
+    String urn = containerKey.asUrnString();
+    // Without instance
+    // "urn:li:container:1929d86c0a92e2d3bb9ba193c8c2b66f";
+    String expectedUrn =
+        TestHelper.generateContainerKeyGuid(
+            "database", Map.of("platform", "test-platform", "database", "test-database"));
+
+    // Assert
+    assertEquals(expectedUrn, urn);
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/FieldPathTest.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/FieldPathTest.java
new file mode 100644
index 0000000000000..55b97f735393c
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/FieldPathTest.java
@@ -0,0 +1,83 @@
+package io.datahubproject.models.util;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class FieldPathTest {
+
+  @Test
+  public void testSimplePathConstruction() {
+    FieldPath path = new FieldPath("field1.field2.field3");
+    assertEquals("field1.field2.field3", path.simplePath());
+    assertEquals("1", path.version());
+    assertEquals("field3", path.leafFieldName());
+    assertEquals(3, path.depth());
+    assertFalse(path.isTopLevel());
+  }
+
+  @Test
+  public void testTopLevelPath() {
+    FieldPath path = new FieldPath("singleField");
+    assertTrue(path.isTopLevel());
+    assertEquals(1, path.depth());
+    assertEquals("singleField", path.leafFieldName());
+    assertEquals("singleField", path.simplePath());
+  }
+
+  @Test
+  public void testVersionExtraction() {
+    FieldPath path = new FieldPath("[version=2.0].field1.field2");
+    assertEquals("2.0", path.version());
+    assertEquals("field1.field2", path.simplePath());
+  }
+
+  @Test
+  public void testComplexPathWithMetadata() {
+    FieldPath path =
+        new FieldPath("[version=2.0][type=Foo].address.[type=union].[type=string].street");
+    assertEquals("2.0", path.version());
+    assertEquals("address.street", path.simplePath());
+    assertEquals("street", path.leafFieldName());
+    assertEquals(2, path.depth());
+  }
+
+  @Test
+  public void testPathWithMultipleMetadataBlocks() {
+    FieldPath path =
+        new FieldPath("[version=3][type=Record][nullable=true].user.details[type=union].name");
+    System.out.println(path.simplePath());
+    assertEquals("3", path.version());
+    assertEquals("user.details.name", path.simplePath());
+    assertEquals("name", path.leafFieldName());
+    assertEquals(3, path.depth());
+  }
+
+  @Test
+  public void testPathWithTrailingMetadata() {
+    FieldPath path = new FieldPath("field1.field2[type=string]");
+    assertEquals("field1.field2", path.simplePath());
+    assertEquals("1", path.version());
+  }
+
+  @Test
+  public void testPathWithOnlyMetadata() {
+    FieldPath path = new FieldPath("[type=string][version=4]");
+    assertEquals("", path.simplePath());
+    assertEquals("4", path.version());
+  }
+
+  @Test
+  public void testDepthCalculationWithComplexPath() {
+    FieldPath path =
+        new FieldPath("[version=2].user.[type=record].address.[type=union].[type=string].street");
+    assertEquals(3, path.depth());
+    assertEquals("user.address.street", path.simplePath());
+  }
+
+  @Test
+  public void testLeafFieldNameWithMetadata() {
+    FieldPath path = new FieldPath("field1.field2[type=string]");
+    assertEquals("field2[type=string]", path.leafFieldName());
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/SchemaKeyTest.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/SchemaKeyTest.java
new file mode 100644
index 0000000000000..7d40491a9b1d8
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/SchemaKeyTest.java
@@ -0,0 +1,62 @@
+package io.datahubproject.models.util;
+
+import static org.junit.Assert.*;
+
+import java.util.Map;
+import org.junit.Test;
+
+public class SchemaKeyTest {
+  @Test
+  public void testSchemaUrnGeneration() {
+    // Test data
+    SchemaKey schemaKey =
+        SchemaKey.builder()
+            .platform("test-platform")
+            .instance("DEV")
+            .database("test-database")
+            .schema("test-schema")
+            .build();
+
+    System.out.println(schemaKey.guidDict());
+
+    // Generate URN
+    String urn = schemaKey.asUrnString();
+    // With instance
+    String expectedUrn =
+        TestHelper.generateContainerKeyGuid(
+            "schema",
+            Map.of(
+                "platform", "test-platform",
+                "instance", "DEV",
+                "database", "test-database",
+                "schema", "test-schema"));
+
+    // Assert
+    assertEquals(expectedUrn, urn);
+  }
+
+  @Test
+  public void testSchemaUrnGenerationNoInstance() {
+    // Test data
+    ContainerKey containerKey =
+        SchemaKey.builder()
+            .platform("test-platform")
+            .database("test-database")
+            .schema("test-schema")
+            .build();
+
+    // Generate URN
+    String urn = containerKey.asUrnString();
+    // Without instance
+    String expectedUrn =
+        TestHelper.generateContainerKeyGuid(
+            "schema",
+            Map.of(
+                "platform", "test-platform",
+                "database", "test-database",
+                "schema", "test-schema"));
+
+    // Assert
+    assertEquals(expectedUrn, urn);
+  }
+}
diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/TestHelper.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/TestHelper.java
new file mode 100644
index 0000000000000..e150ceda16262
--- /dev/null
+++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/TestHelper.java
@@ -0,0 +1,114 @@
+package io.datahubproject.models.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStreamReader;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+public class TestHelper {
+
+  private static final String TEST_RESOURCES_DIR = "src/test/resources";
+  //  private static final String TEMP_OUTPUT_DIR = "build/test-outputs";
+  private static final String PYTHON_SCRIPT = "scripts/container_key_guid_generator.py";
+  private static final String VENV_PATH =
+      "../../../metadata-ingestion/venv"; // Adjust this path to your venv location
+
+  public static void setup() {
+    // Create output directory if it doesn't exist
+    //    new File(TEMP_OUTPUT_DIR).mkdirs();
+
+    // Verify venv exists
+    if (!new File(VENV_PATH).exists()) {
+      throw new RuntimeException("Virtual environment not found at " + VENV_PATH);
+    }
+  }
+
+  private static ProcessBuilder createPythonProcessBuilder(String... args) {
+    ProcessBuilder pb;
+    String os = System.getProperty("os.name").toLowerCase();
+
+    if (os.contains("windows")) {
+      // Windows paths
+      String pythonPath = Paths.get(VENV_PATH, "Scripts", "python").toString();
+      pb =
+          new ProcessBuilder(
+              Stream.concat(Stream.of(pythonPath), Stream.of(args)).toArray(String[]::new));
+    } else {
+      // Unix-like paths
+      String pythonPath = Paths.get(VENV_PATH, "bin", "python").toString();
+      pb =
+          new ProcessBuilder(
+              Stream.concat(Stream.of(pythonPath), Stream.of(args)).toArray(String[]::new));
+    }
+
+    // Add virtual environment to PYTHONPATH
+    Map<String, String> env = pb.environment();
+    String sitePkgPath =
+        Paths.get(
+                VENV_PATH,
+                os.contains("windows") ? "Lib/site-packages" : "lib/python3.x/site-packages")
+            .toString();
+
+    String pythonPath = env.getOrDefault("PYTHONPATH", "");
+    env.put("PYTHONPATH", pythonPath + File.pathSeparator + sitePkgPath);
+
+    return pb;
+  }
+
+  /**
+   * Executes the container key GUID generator Python script with the given parameters
+   *
+   * @param containerType The type of container (e.g., "database")
+   * @param parameters Map of parameter key-value pairs
+   * @return The generated URN as a string
+   * @throws RuntimeException if the Python script execution fails
+   */
+  public static String generateContainerKeyGuid(
+      String containerType, Map<String, String> parameters) {
+    List<String> command = new ArrayList<>();
+    command.add(PYTHON_SCRIPT);
+    command.add("--container-type");
+    command.add(containerType);
+
+    // Add each parameter as -p key=value
+    for (Map.Entry<String, String> entry : parameters.entrySet()) {
+      command.add("-p");
+      command.add(entry.getKey() + "=" + entry.getValue());
+    }
+
+    try {
+      System.out.println("Executing Python script: " + String.join(" ", command));
+      String[] commandArray = command.toArray(new String[0]);
+
+      Process process = createPythonProcessBuilder(commandArray).start();
+
+      // Read the output
+      BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+      String line = reader.readLine();
+
+      // Wait for the process to complete
+      int exitCode = process.waitFor();
+
+      if (exitCode != 0) {
+        // Read error stream if the script failed
+        BufferedReader errorReader =
+            new BufferedReader(new InputStreamReader(process.getErrorStream()));
+        StringBuilder errorMessage = new StringBuilder();
+        String errorLine;
+        while ((errorLine = errorReader.readLine()) != null) {
+          errorMessage.append(errorLine).append("\n");
+        }
+        throw new RuntimeException(
+            "Python script failed with exit code " + exitCode + ": " + errorMessage);
+      }
+
+      return line != null ? line.trim() : "";
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to execute Python script", e);
+    }
+  }
+}
diff --git a/metadata-integration/java/datahub-event/build.gradle b/metadata-integration/java/datahub-event/build.gradle
index 395065404d1db..752e95656bcf6 100644
--- a/metadata-integration/java/datahub-event/build.gradle
+++ b/metadata-integration/java/datahub-event/build.gradle
@@ -1,13 +1,13 @@
 plugins {
   id("com.palantir.git-version") apply false
   id 'java'
-  id 'com.github.johnrengelman.shadow'
-  id 'jacoco'
+  id 'com.gradleup.shadow'
   id 'signing'
   id 'io.codearte.nexus-staging'
   id 'maven-publish'
 }
 
+apply from: "../../../gradle/coverage/java-coverage.gradle"
 apply from: "../versioning.gradle"
 
 dependencies {
@@ -18,6 +18,7 @@ dependencies {
   implementation externalDependency.jacksonDataBind
   runtimeOnly externalDependency.jna
 
+  compileOnly externalDependency.swaggerAnnotations
   compileOnly externalDependency.lombok
   annotationProcessor externalDependency.lombok
   // VisibleForTesting
@@ -29,10 +30,6 @@ dependencies {
   testRuntimeOnly externalDependency.logbackClassicJava8
 }
 
-jacocoTestReport {
-  dependsOn test // tests are required to run before generating the report
-}
-
 task copyAvroSchemas {
   dependsOn(':metadata-events:mxe-schemas:renameNamespace')
   copy {
@@ -47,7 +44,6 @@ test {
   // to avoid simultaneous executions of tests when complete build is run
   mustRunAfter(":metadata-io:test")
   useJUnit()
-  finalizedBy jacocoTestReport
 }
 
 // task sourcesJar(type: Jar) {
diff --git a/metadata-integration/java/datahub-protobuf/build.gradle b/metadata-integration/java/datahub-protobuf/build.gradle
index 8bc71c5da8f4a..748ab2ef8afeb 100644
--- a/metadata-integration/java/datahub-protobuf/build.gradle
+++ b/metadata-integration/java/datahub-protobuf/build.gradle
@@ -3,12 +3,12 @@ plugins {
     id "application"
 }
 apply plugin: 'java'
-apply plugin: 'jacoco'
-apply plugin: 'com.github.johnrengelman.shadow'
+apply plugin: 'com.gradleup.shadow'
 apply plugin: 'signing'
 apply plugin: 'io.codearte.nexus-staging'
 apply plugin: 'maven-publish'
 apply from: '../versioning.gradle'
+apply from: '../../../gradle/coverage/java-coverage.gradle'
 
 jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
 
@@ -57,13 +57,6 @@ task compileProtobuf {
     }
 }
 
-jacocoTestReport {
-    dependsOn test // tests are required to run before generating the report
-}
-
-test.finalizedBy jacocoTestReport
-
-
 task checkShadowJar(type: Exec) {
    commandLine 'sh', '-c', 'scripts/check_jar.sh'
 }
@@ -202,4 +195,4 @@ nexusStaging {
     password = System.getenv("NEXUS_PASSWORD")
 }
 
-startScripts.dependsOn shadowJar
\ No newline at end of file
+startScripts.dependsOn shadowJar
diff --git a/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh b/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh
index bd0c28f0f8698..66c70f0b85769 100755
--- a/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh
+++ b/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh
@@ -44,7 +44,9 @@ jar -tvf $jarFile |\
       grep -v "mime.types" |\
       grep -v "com/ibm/.*" |\
       grep -v "org/glassfish/" |\
-      grep -v "LICENSE"
+      grep -v "LICENSE" |\
+      grep -v "org/apache/avro" |\
+      grep -v "org/apache"
 
 if [ $? -ne 0 ]; then
   echo "✅ No unexpected class paths found in ${jarFile}"
diff --git a/metadata-integration/java/datahub-schematron/README.md b/metadata-integration/java/datahub-schematron/README.md
new file mode 100644
index 0000000000000..0dc1c2b9c7455
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/README.md
@@ -0,0 +1,73 @@
+# SchemaTron (Incubating)
+
+> ⚠️ This is an incubating project in draft status. APIs and functionality may change significantly between releases.
+
+SchemaTron is a schema translation toolkit that converts between various schema formats and DataHub's native schema representation. It currently provides robust support for Apache Avro schema translation with a focus on complex schema structures including unions, arrays, maps, and nested records.
+
+## Modules
+
+### CLI Module
+
+Command-line interface for converting schemas and emitting them to DataHub.
+
+```bash
+# Execute from this directory
+../../../gradlew :metadata-integration:java:datahub-schematron:cli:run --args="-i cli/src/test/resources/FlatUser.avsc"
+```
+
+#### CLI Options
+
+- `-i, --input`: Input schema file or directory path
+- `-p, --platform`: Data platform name (default: "avro")
+- `-s, --server`: DataHub server URL (default: "http://localhost:8080")
+- `-t, --token`: DataHub access token
+- `--sink`: Output sink - "rest" or "file" (default: "rest")
+- `--output-file`: Output file path when using file sink (default: "metadata.json")
+
+### Library Module
+
+Core translation logic and models for schema conversion. Features include:
+
+- Support for complex Avro schema structures:
+  - Union types with multiple record options
+  - Nested records and arrays
+  - Optional fields with defaults
+  - Logical types (date, timestamp, etc.)
+  - Maps with various value types
+  - Enum types
+  - Custom metadata and documentation
+
+- Comprehensive path handling for schema fields
+- DataHub-compatible metadata generation
+- Schema fingerprinting and versioning
+
+## Example Schema Support
+
+The library can handle sophisticated schema structures including:
+
+- Customer profiles with multiple identification types (passport, driver's license, national ID)
+- Contact information with primary and alternative contact methods
+- Address validation with verification metadata
+- Subscription history tracking
+- Flexible preference and metadata storage
+- Tagged customer attributes
+
+## Development
+
+The project includes extensive test coverage through:
+
+- Unit tests for field path handling
+- Schema translation comparison tests
+- Integration tests with Python reference implementation
+
+Test resources include example schemas demonstrating various Avro schema features and edge cases.
+
+## Contributing
+
+As this is an incubating project, we welcome contributions and feedback on:
+
+- Additional schema format support
+- Improved handling of complex schema patterns
+- Enhanced metadata translation
+- Documentation and examples
+- Test coverage
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/cli/build.gradle b/metadata-integration/java/datahub-schematron/cli/build.gradle
new file mode 100644
index 0000000000000..1711ff947c2d1
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/build.gradle
@@ -0,0 +1,110 @@
+plugins {
+    id "application"
+}
+apply plugin: 'java'
+apply plugin: 'jacoco'
+
+ext {
+    javaMainClass = "io.datahubproject.schematron.cli.SchemaTron"
+}
+
+application {
+    mainClassName = javaMainClass
+}
+
+dependencies {
+    // Existing dependencies remain unchanged
+    implementation 'info.picocli:picocli:4.7.5'
+    annotationProcessor 'info.picocli:picocli-codegen:4.7.5'
+    implementation 'ch.qos.logback:logback-classic:1.2.11'
+    implementation 'ch.qos.logback:logback-core:1.2.11'
+    implementation project(':metadata-integration:java:datahub-client')
+    implementation project(':metadata-integration:java:datahub-schematron:lib')
+    implementation externalDependency.avro
+    compileOnly externalDependency.lombok
+    annotationProcessor externalDependency.lombok
+
+    // Test dependencies
+    testImplementation externalDependency.testng
+    testImplementation externalDependency.mockito
+}
+
+test {
+    useTestNG()
+    
+    testLogging {
+        events "passed", "skipped", "failed"
+        exceptionFormat "full"
+        showStandardStreams = true
+    }
+    
+    systemProperty 'python.venv.path', System.getProperty('python.venv.path', '../venv')
+}
+
+task validatePythonEnv {
+    doFirst {
+        def venvPath = System.getProperty('python.venv.path', '../../../../metadata-ingestion/venv')
+        def isWindows = System.getProperty('os.name').toLowerCase().contains('windows')
+        def pythonExe = isWindows ? "${venvPath}/Scripts/python.exe" : "${venvPath}/bin/python"
+        
+        def result = exec {
+            commandLine pythonExe, "-c", "import sys; print(sys.executable)"
+            ignoreExitValue = true
+            standardOutput = new ByteArrayOutputStream()
+            errorOutput = new ByteArrayOutputStream()
+        }
+        
+        if (result.exitValue != 0) {
+            throw new GradleException("Python virtual environment not properly set up at ${venvPath}")
+        }
+    }
+}
+
+test.dependsOn tasks.getByPath(":metadata-ingestion:installDev")
+
+jacocoTestReport {
+    dependsOn test
+}
+
+test.finalizedBy jacocoTestReport
+
+task updateGoldenFiles {
+    dependsOn validatePythonEnv
+    doLast {
+        def venvPath = System.getProperty('python.venv.path', '../../../../metadata-ingestion/venv')
+        def isWindows = System.getProperty('os.name').toLowerCase().contains('windows')
+        def pythonExe = isWindows ? "${venvPath}/Scripts/python.exe" : "${venvPath}/bin/python"
+        def diffsDir = new File('src/test/resources/diffs')
+        
+        if (!diffsDir.exists()) {
+            throw new GradleException("Diffs directory not found at ${diffsDir.absolutePath}")
+        }
+        
+        // Find all json files in the diffs directory
+        diffsDir.listFiles().findAll { it.name.endsWith('_diff.json') }.each { diffFile ->
+            def baseName = diffFile.name.replace('_diff.json', '')
+            def pythonOutput = "build/test-outputs/${baseName}_python.json"
+            def javaOutput = "build/test-outputs/${baseName}_java.json"
+            
+            println "Updating golden file for ${baseName}..."
+            
+            exec {
+                commandLine pythonExe,
+                        'scripts/mce_diff.py',
+                        '--update-golden-diff',
+                        '--golden-diff-file',
+                        diffFile.absolutePath,
+                        pythonOutput,
+                        javaOutput
+                ignoreExitValue = true
+                standardOutput = new ByteArrayOutputStream()
+                errorOutput = new ByteArrayOutputStream()
+            }
+        }
+    }
+}
+
+configurations {
+    provided
+    implementation.extendsFrom provided
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/cli/scripts/avro_schema_to_mce.py b/metadata-integration/java/datahub-schematron/cli/scripts/avro_schema_to_mce.py
new file mode 100644
index 0000000000000..38a90bc331842
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/scripts/avro_schema_to_mce.py
@@ -0,0 +1,94 @@
+from datahub.ingestion.extractor.schema_util import AvroToMceSchemaConverter
+from avro.schema import parse as parse_avro, RecordSchema
+from datahub.emitter.synchronized_file_emitter import SynchronizedFileEmitter
+import datahub.metadata.schema_classes as models
+import click
+from datahub.emitter.mce_builder import make_data_platform_urn, make_dataset_urn
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+import os
+import hashlib
+from datahub.ingestion.graph.client import get_default_graph
+
+
+def get_schema_hash(schema):
+    # Convert schema to string if it isn't already
+    schema_str = str(schema)
+
+    # Create MD5 hash
+    schema_hash = hashlib.md5(schema_str.encode("utf-8")).hexdigest()
+
+    return schema_hash
+
+
+@click.command(name="avro2datahub")
+@click.option("--input-file", "-i", type=click.Path(exists=True), required=True)
+@click.option("--platform", type=str, required=True)
+@click.option("--output-file", "-o", type=click.Path(), default="metadata.py.json")
+@click.option("--to-file", "-f", is_flag=True, default=True)
+@click.option("--to-server", "-s", is_flag=True, default=False)
+def generate_schema_file_from_avro_schema(
+    input_file: str, platform: str, output_file: str, to_file: bool, to_server: bool
+):
+    avro_schema_file = input_file
+    output_file_name = output_file
+    platform_urn = make_data_platform_urn(platform)
+    converter = AvroToMceSchemaConverter(is_key_schema=False)
+
+    # Delete the output file if it exists
+    if os.path.exists(output_file_name):
+        os.remove(output_file_name)
+
+    with open(avro_schema_file) as f:
+        raw_string = f.read()
+        avro_schema = parse_avro(raw_string)
+        # Get fingerprint bytes
+        canonical_form = avro_schema.canonical_form
+        print(
+            f"Schema canonical form: Length ({len(canonical_form)}); {canonical_form}"
+        )
+        md5_bytes = avro_schema.fingerprint("md5")
+        # Convert to hex string
+        avro_schema_hash = md5_bytes.hex()
+        assert isinstance(
+            avro_schema, RecordSchema
+        ), "This command only works for Avro records"
+        dataset_urn = make_dataset_urn(
+            platform=platform_urn,
+            name=(
+                f"{avro_schema.namespace}.{avro_schema.name}"
+                if avro_schema.namespace
+                else avro_schema.name
+            ),
+        )
+        schema_fields = [
+            f for f in converter.to_mce_fields(avro_schema, is_key_schema=False)
+        ]
+        schema_metadata = models.SchemaMetadataClass(
+            schemaName=avro_schema.name,
+            platform=platform_urn,
+            version=0,
+            hash=avro_schema_hash,
+            platformSchema=models.OtherSchemaClass(rawSchema=raw_string),
+            fields=schema_fields,
+        )
+        assert schema_metadata.validate()
+        if to_file:
+            with SynchronizedFileEmitter(output_file_name) as file_emitter:
+                file_emitter.emit(
+                    MetadataChangeProposalWrapper(
+                        entityUrn=dataset_urn, aspect=schema_metadata
+                    )
+                )
+        if to_server:
+            with get_default_graph() as graph:
+                graph.emit(
+                    MetadataChangeProposalWrapper(
+                        entityUrn=dataset_urn, aspect=schema_metadata
+                    )
+                )
+
+    print(f"Wrote metadata to {output_file}")
+
+
+if __name__ == "__main__":
+    generate_schema_file_from_avro_schema()
diff --git a/metadata-integration/java/datahub-schematron/cli/scripts/mce_diff.py b/metadata-integration/java/datahub-schematron/cli/scripts/mce_diff.py
new file mode 100644
index 0000000000000..37ba11138610c
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/scripts/mce_diff.py
@@ -0,0 +1,345 @@
+import json
+from typing import Dict, Any, Optional, Tuple
+import click
+
+import json
+from typing import Dict, Any
+
+
+def diff_lists(list1, list2):
+    """
+    Compare two lists element by element and return their differences.
+
+    Args:
+        list1 (list): First list to compare
+        list2 (list): Second list to compare
+
+    Returns:
+        dict: A dictionary containing the differences
+    """
+    result = {"added": {}, "removed": {}, "modified": set(), "modified_details": {}}
+
+    if len(list1) != len(list2):
+        # Let's first line up the elements that are common to both lists using
+        # the fieldPath as the key if it exists
+        if "fieldPath" in list1[0]:
+            list1_dict = {field["fieldPath"]: field for field in list1}
+            list2_dict = {field["fieldPath"]: field for field in list2}
+            common_keys = set(list1_dict.keys()) & set(list2_dict.keys())
+            list1 = [list1_dict[key] for key in common_keys]
+            list2 = [list2_dict[key] for key in common_keys]
+            list1.extend(
+                [list1_dict[key] for key in set(list1_dict.keys()) - common_keys]
+            )
+            list2.extend(
+                [list2_dict[key] for key in set(list2_dict.keys()) - common_keys]
+            )
+
+    # Handle added elements (if list2 is longer)
+    if len(list2) > len(list1):
+        for i in range(len(list1), len(list2)):
+            if "fieldPath" in list2[i]:
+                result["added"][list2[i]["fieldPath"]] = list2[i]
+            else:
+                result["added"][str(i)] = list2[i]
+
+    # Handle removed elements (if list1 is longer)
+    if len(list1) > len(list2):
+        for i in range(len(list2), len(list1)):
+            if "fieldPath" in list1[i]:
+                result["removed"][list1[i]["fieldPath"]] = list1[i]
+            else:
+                result["removed"][str(i)] = list1[i]
+
+    # Compare common indices
+    for i in range(min(len(list1), len(list2))):
+        value1 = list1[i]
+        value2 = list2[i]
+
+        if type(value1) != type(value2):
+            result["modified"].add(str(i))
+            result["modified_details"][str(i)] = {"before": value1, "after": value2}
+        elif isinstance(value1, dict) and isinstance(value2, dict):
+            nested_diff = diff_dicts(
+                value1, value2, identifier=value1.get("fieldPath", i)
+            )
+            if any(nested_diff.values()):
+                result["modified"].add(value1.get("fieldPath", i))
+                result["modified_details"][value1.get("fieldPath", i)] = nested_diff
+        elif isinstance(value1, list) and isinstance(value2, list):
+            nested_diff = diff_lists(value1, value2)
+            if any(nested_diff.values()):
+                result["modified"].add(str(i))
+                result["modified_details"][str(i)] = nested_diff
+        elif value1 != value2:
+            result["modified"].add(str(i))
+            result["modified_details"][str(i)] = {
+                "before": value1,
+                "after": value2,
+                "identifier": i,
+            }
+
+    return result
+
+
+def diff_schema_field(field1_dict, field2_dict):
+
+    from datahub.metadata.schema_classes import SchemaFieldClass
+
+    field1 = SchemaFieldClass.from_obj(field1_dict)
+    field2 = SchemaFieldClass.from_obj(field2_dict)
+
+    # Initialize result structure
+    result = {"added": {}, "removed": {}, "modified": set(), "modified_details": {}}
+
+    result = {}
+    if field1.fieldPath != field2.fieldPath:
+        result["fieldPath"] = {"before": field1.fieldPath, "after": field2.fieldPath}
+
+    if field1.type != field2.type:
+        result["type"] = {
+            "before": field1.type,
+            "after": field2.type,
+            "identifier": field1.fieldPath,
+        }
+
+    if field1.description != field2.description:
+        result["description"] = {
+            "before": field1.description,
+            "after": field2.description,
+            "identifier": field1.fieldPath,
+        }
+
+    if field1.nullable != field2.nullable:
+        result["nullable"] = {
+            "before": field1.nullable,
+            "after": field2.nullable,
+            "identifier": field1.fieldPath,
+        }
+
+    return result
+
+
+def diff_schema_metadata(schema1_dict, schema2_dict):
+
+    ignored_for_diff = [
+        "created",
+        "modified",
+        "hash",
+        "platformSchema",
+        "lastModified",
+    ]  # TODO: Reduce this list
+
+    for key in ignored_for_diff:
+        schema1_dict.pop(key, None)
+        schema2_dict.pop(key, None)
+
+    return diff_dicts(schema1_dict, schema2_dict)
+
+
+def is_empty_diff(diff_dict) -> bool:
+    if diff_dict.keys() == EMPTY_DIFF().keys():
+        for key in diff_dict:
+            if diff_dict[key]:
+                return False
+        return True
+    return False
+
+
+def format_diff(diff_dict) -> Any:
+    if isinstance(diff_dict, set):
+        diff_dict = sorted(list([x for x in diff_dict]))
+    elif isinstance(diff_dict, dict):
+        for key in diff_dict:
+            diff_dict[key] = format_diff(diff_dict[key])
+    return diff_dict
+
+
+def EMPTY_DIFF():
+    return {
+        "added": {},
+        "removed": {},
+        "modified": set(),
+        "modified_details": {},
+    }
+
+
+def diff_dicts(dict1, dict2, identifier=None):
+    """
+    Compare two dictionaries recursively and return their differences.
+
+    Args:
+        dict1 (dict): First dictionary to compare
+        dict2 (dict): Second dictionary to compare
+
+    Returns:
+        dict: A dictionary containing the differences with the following structure:
+            {
+                'added': Keys present in dict2 but not in dict1,
+                'removed': Keys present in dict1 but not in dict2,
+                'modified': Keys present in both but with different values,
+                'modified_details': Detailed before/after values for modified keys
+            }
+    """
+    if "nullable" in dict1:
+        # Assume this is a SchemaFieldClass
+        return diff_schema_field(dict1, dict2)
+
+    if "hash" in dict1:
+        # Assume this is a schema metadata class
+        return diff_schema_metadata(dict1, dict2)
+
+    dict1_keys = set(dict1.keys())
+    dict2_keys = set(dict2.keys())
+
+    # Find keys that were added, removed, or modified
+    added_keys = dict2_keys - dict1_keys
+    removed_keys = dict1_keys - dict2_keys
+    common_keys = dict1_keys & dict2_keys
+
+    # Initialize result structure
+    result = EMPTY_DIFF()
+    # Handle added keys
+    for key in added_keys:
+        result["added"][key] = dict2[key]
+
+    # Handle removed keys
+    for key in removed_keys:
+        result["removed"][key] = dict1[key]
+
+    # Check common keys for modifications
+    for key in common_keys:
+        value1 = dict1[key]
+        value2 = dict2[key]
+
+        # If both values are dictionaries, recurse
+        if isinstance(value1, dict) and isinstance(value2, dict):
+            nested_diff = diff_dicts(
+                value1, value2, identifier=value1.get("fieldPath", key)
+            )
+            if any(nested_diff.values()):  # If there are any differences
+                result["modified"].add(key)
+                result["modified_details"][key] = nested_diff
+        # If both values are lists, compare them element by element
+        elif isinstance(value1, list) and isinstance(value2, list):
+            nested_diff = diff_lists(value1, value2)
+            if any(nested_diff.values()):
+                result["modified"].add(key)
+                result["modified_details"][key] = nested_diff
+        # Otherwise compare directly
+        elif value1 != value2:
+            result["modified"].add(key)
+            result["modified_details"][key] = {
+                "before": value1,
+                "after": value2,
+                "identifier": identifier,
+            }
+
+    return result
+
+
+def process_single_element(element) -> Tuple[str, str, Dict[str, Any]]:
+    if "entityUrn" in element:
+        entity = element["entityUrn"]
+    else:
+        raise Exception("Element does not have an entityUrn key")
+    if "aspectName" in element:
+        aspect = element["aspectName"]
+    else:
+        raise Exception("Element does not have an aspectName key")
+    if "aspect" in element:
+        if "json" in element["aspect"]:
+            return entity, aspect, element["aspect"]["json"]
+        elif "value" in element["aspect"]:
+            json_value = json.loads(element["aspect"]["value"])
+            return entity, aspect, json_value
+        else:
+            raise Exception("Element does not have a json or value key")
+    else:
+        raise Exception("Element does not have an aspect key")
+
+
+def process_element_with_dict(element, global_dict):
+    entity, aspect, data = process_single_element(element)
+    if entity not in global_dict:
+        global_dict[entity] = {}
+    if aspect not in global_dict[entity]:
+        global_dict[entity][aspect] = data
+    else:
+        # breakpoint()
+        raise Exception("Duplicate aspect found")
+
+
+@click.command("compute_diff")
+@click.argument("input_file_1", type=click.Path(exists=True))
+@click.argument("input_file_2", type=click.Path(exists=True))
+@click.option("--golden-diff-file", type=click.Path(), default=None)
+@click.option("--update-golden-diff", is_flag=True)
+def compute_diff(
+    input_file_1: str,
+    input_file_2: str,
+    golden_diff_file: Optional[str] = None,
+    update_golden_diff: bool = False,
+):
+
+    # Read the files into json objects and compare them
+    # If they are the same, exit 0
+    # If they are different, exit 1
+    file_1_mcps = {}
+    with open(input_file_1) as file1:
+        data1 = json.load(file1)
+        assert isinstance(data1, list)
+        for element in data1:
+            process_element_with_dict(element, file_1_mcps)
+    print(f"Processed {len(file_1_mcps)} elements from file {input_file_1}")
+
+    file_2_mcps = {}
+    with open(input_file_2) as file2:
+        data2 = json.load(file2)
+        assert isinstance(data2, list)
+        for element in data2:
+            process_element_with_dict(element, file_2_mcps)
+
+    print(f"Processed {len(file_2_mcps)} elements from file {input_file_2}")
+
+    if golden_diff_file and not update_golden_diff:
+        with open(golden_diff_file) as golden_diff:
+            golden_diff_data = json.load(golden_diff)
+    else:
+        golden_diff_data = None
+
+    computed_diff_data = {}
+
+    assert len(file_1_mcps) == len(file_2_mcps)
+    for entity in file_1_mcps:
+        assert entity in file_2_mcps
+        assert len(file_1_mcps[entity]) == len(file_2_mcps[entity])
+        for aspect in file_1_mcps[entity]:
+            assert aspect in file_2_mcps[entity]
+            aspect_diff = diff_dicts(
+                file_1_mcps[entity][aspect], file_2_mcps[entity][aspect]
+            )
+            if golden_diff_data:
+                assert aspect in golden_diff_data[entity]
+                assert format_diff(aspect_diff) == golden_diff_data[entity][aspect], (
+                    f"Computed difference is {json.dumps(format_diff(aspect_diff), indent=2)}\n"
+                    f"Expected difference is {json.dumps(golden_diff_data[entity][aspect], indent=2)}"
+                )
+
+            else:
+                if update_golden_diff:
+                    if entity not in computed_diff_data:
+                        computed_diff_data[entity] = {}
+                    computed_diff_data[entity][aspect] = format_diff(aspect_diff)
+                else:
+                    assert is_empty_diff(
+                        aspect_diff
+                    ), f"Difference is {json.dumps(format_diff(aspect_diff), indent=2)}"
+
+    if update_golden_diff:
+        with open(golden_diff_file, "w") as golden_diff:
+            json.dump(computed_diff_data, golden_diff, indent=2, sort_keys=True)
+
+
+if __name__ == "__main__":
+    compute_diff()
diff --git a/metadata-integration/java/datahub-schematron/cli/src/main/java/io/datahubproject/schematron/cli/SchemaTron.java b/metadata-integration/java/datahub-schematron/cli/src/main/java/io/datahubproject/schematron/cli/SchemaTron.java
new file mode 100644
index 0000000000000..d8e4a43cfa8fb
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/src/main/java/io/datahubproject/schematron/cli/SchemaTron.java
@@ -0,0 +1,147 @@
+package io.datahubproject.schematron.cli;
+
+import com.linkedin.common.FabricType;
+import com.linkedin.common.urn.DataPlatformUrn;
+import com.linkedin.common.urn.DatasetUrn;
+import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.schema.SchemaField;
+import com.linkedin.schema.SchemaMetadata;
+import datahub.client.Emitter;
+import datahub.client.file.FileEmitter;
+import datahub.client.file.FileEmitterConfig;
+import datahub.client.rest.RestEmitter;
+import datahub.event.MetadataChangeProposalWrapper;
+import io.datahubproject.schematron.converters.avro.AvroSchemaConverter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+import java.util.stream.Stream;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.avro.Schema;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+@Slf4j
+@Command(
+    name = "schema-translator",
+    description = "Converts schemas to DataHub format and emits them",
+    mixinStandardHelpOptions = true)
+public class SchemaTron implements Callable<Integer> {
+
+  @Option(
+      names = {"-i", "--input"},
+      description = "Input schema file or directory")
+  private String input;
+
+  @Option(
+      names = {"-s", "--server"},
+      description = "DataHub server URL",
+      required = false,
+      defaultValue = "http://localhost:8080")
+  private String server;
+
+  @Option(
+      names = {"-t", "--token"},
+      description = "DataHub access token",
+      required = false,
+      defaultValue = "")
+  private String token;
+
+  @Option(
+      names = {"-p", "--platform"},
+      description = "Data platform name",
+      defaultValue = "avro")
+  private String platform;
+
+  @Option(
+      names = {"--sink"},
+      description = "DataHub sink name",
+      defaultValue = "rest")
+  private String sink;
+
+  @Option(
+      names = {"--output-file"},
+      description = "Output file for the emitted metadata",
+      defaultValue = "metadata.json")
+  private String outputFile;
+
+  private final AvroSchemaConverter schemaConverter = AvroSchemaConverter.builder().build();
+
+  @Override
+  public Integer call() throws Exception {
+
+    Emitter emitter;
+    if (sink.equals("rest")) {
+      emitter = RestEmitter.create(b -> b.server(server).token(token));
+    } else if (sink.equals("file")) {
+      emitter = new FileEmitter(FileEmitterConfig.builder().fileName(outputFile).build());
+    } else {
+      throw new IllegalArgumentException("Unsupported sink: " + sink);
+    }
+
+    try {
+      // Process input files
+      Stream<Path> inputFiles;
+      Path inputPath = Path.of(input);
+      if (Files.isDirectory(inputPath)) {
+        inputFiles = Files.walk(inputPath).filter(p -> p.toString().endsWith(".avsc"));
+      } else {
+        inputFiles = Stream.of(inputPath);
+      }
+
+      // Process each file
+      inputFiles.forEach(
+          filePath -> {
+            try {
+              // Read and parse Avro schema
+              String schemaStr = Files.readString(filePath);
+              Schema avroSchema = new Schema.Parser().parse(schemaStr);
+
+              // Convert to DataHub schema
+              boolean isKeySchema = false;
+              boolean isDefaultNullable = false;
+              SchemaMetadata schemaMetadata =
+                  schemaConverter.toDataHubSchema(
+                      avroSchema,
+                      isKeySchema,
+                      isDefaultNullable,
+                      new DataPlatformUrn(platform),
+                      null);
+              log.info("Generated {} fields", schemaMetadata.getFields().size());
+              for (SchemaField field : schemaMetadata.getFields()) {
+                log.debug("Field path: {}", field.getFieldPath());
+              }
+
+              DatasetUrn datasetUrn =
+                  new DatasetUrn(
+                      new DataPlatformUrn(platform), avroSchema.getFullName(), FabricType.PROD);
+
+              MetadataChangeProposalWrapper<SchemaMetadata> wrapper =
+                  new MetadataChangeProposalWrapper(
+                      "dataset",
+                      datasetUrn.toString(),
+                      ChangeType.UPSERT,
+                      schemaMetadata,
+                      "schemaMetadata");
+
+              // Emit to DataHub
+              emitter.emit(wrapper, null).get();
+              log.info("Emitted schema for {}", datasetUrn);
+            } catch (Exception e) {
+              System.err.println("Error processing file: " + filePath);
+              e.printStackTrace();
+            }
+          });
+
+      return 0;
+    } finally {
+      emitter.close();
+    }
+  }
+
+  public static void main(String[] args) {
+    int exitCode = new CommandLine(new SchemaTron()).execute(args);
+    System.exit(exitCode);
+  }
+}
diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/java/io/datahubproject/schematron/SchemaTranslatorTest.java b/metadata-integration/java/datahub-schematron/cli/src/test/java/io/datahubproject/schematron/SchemaTranslatorTest.java
new file mode 100644
index 0000000000000..bb11beb00729e
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/src/test/java/io/datahubproject/schematron/SchemaTranslatorTest.java
@@ -0,0 +1,149 @@
+package io.datahubproject.schematron;
+
+import static org.testng.Assert.assertEquals;
+
+import io.datahubproject.schematron.cli.SchemaTron;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+import picocli.CommandLine;
+
+public class SchemaTranslatorTest {
+  private static final String TEST_RESOURCES_DIR = "src/test/resources";
+  private static final String TEMP_OUTPUT_DIR = "build/test-outputs";
+  private static final String PYTHON_SCRIPT = "scripts/avro_schema_to_mce.py";
+  private static final String DIFF_SCRIPT = "scripts/mce_diff.py";
+  private static final String VENV_PATH =
+      "../../../../metadata-ingestion/venv"; // Adjust this path to your venv location
+
+  @BeforeClass
+  public static void setup() {
+    // Create output directory if it doesn't exist
+    new File(TEMP_OUTPUT_DIR).mkdirs();
+
+    // Verify venv exists
+    if (!new File(VENV_PATH).exists()) {
+      throw new RuntimeException("Virtual environment not found at " + VENV_PATH);
+    }
+  }
+
+  @DataProvider(name = "schemaFiles")
+  public Object[][] getSchemaFiles() throws Exception {
+    List<Path> schemaFiles =
+        Files.walk(Paths.get(TEST_RESOURCES_DIR))
+            .filter(path -> path.toString().endsWith(".avsc"))
+            .collect(Collectors.toList());
+
+    Object[][] testData = new Object[schemaFiles.size()][1];
+    for (int i = 0; i < schemaFiles.size(); i++) {
+      testData[i][0] = schemaFiles.get(i);
+    }
+    return testData;
+  }
+
+  @Test(dataProvider = "schemaFiles")
+  public void testSchemaTranslations(Path schemaFile) throws Exception {
+    compareTranslations(schemaFile);
+  }
+
+  private ProcessBuilder createPythonProcessBuilder(String... args) {
+    ProcessBuilder pb;
+    String os = System.getProperty("os.name").toLowerCase();
+
+    if (os.contains("windows")) {
+      // Windows paths
+      String pythonPath = Paths.get(VENV_PATH, "Scripts", "python").toString();
+      pb =
+          new ProcessBuilder(
+              Stream.concat(Stream.of(pythonPath), Stream.of(args)).toArray(String[]::new));
+    } else {
+      // Unix-like paths
+      String pythonPath = Paths.get(VENV_PATH, "bin", "python").toString();
+      pb =
+          new ProcessBuilder(
+              Stream.concat(Stream.of(pythonPath), Stream.of(args)).toArray(String[]::new));
+    }
+
+    // Add virtual environment to PYTHONPATH
+    Map<String, String> env = pb.environment();
+    String sitePkgPath =
+        Paths.get(
+                VENV_PATH,
+                os.contains("windows") ? "Lib/site-packages" : "lib/python3.x/site-packages")
+            .toString();
+
+    String pythonPath = env.getOrDefault("PYTHONPATH", "");
+    env.put("PYTHONPATH", pythonPath + File.pathSeparator + sitePkgPath);
+
+    return pb.inheritIO();
+  }
+
+  private void compareTranslations(Path schemaFile) throws Exception {
+    String baseName = schemaFile.getFileName().toString().replace(".avsc", "");
+    String javaOutput = TEMP_OUTPUT_DIR + "/" + baseName + "_java.json";
+    String pythonOutput = TEMP_OUTPUT_DIR + "/" + baseName + "_python.json";
+    String diffFile = schemaFile.getParent().toString() + "/diffs/" + baseName + "_diff.json";
+
+    // Test if diffFile exists
+    File diff = new File(diffFile);
+    if (!diff.exists()) {
+      diffFile = null;
+    }
+
+    // Run Python translator
+    Process pythonProcess =
+        createPythonProcessBuilder(
+                PYTHON_SCRIPT,
+                "--platform",
+                "datahub",
+                "--input-file",
+                schemaFile.toString(),
+                "--output-file",
+                pythonOutput)
+            .inheritIO()
+            .start();
+
+    int pythonExitCode = pythonProcess.waitFor();
+    assertEquals(pythonExitCode, 0, "Python translation failed");
+
+    // Run Java translator directly using SchemaTron
+    SchemaTron schemaTron = new SchemaTron();
+    int javaExitCode =
+        new CommandLine(schemaTron)
+            .execute(
+                "-i",
+                schemaFile.toAbsolutePath().toString(),
+                "--sink",
+                "file",
+                "--output-file",
+                javaOutput,
+                "--platform",
+                "datahub");
+
+    assertEquals(javaExitCode, 0, "Java translation failed");
+
+    // Compare outputs
+    // if diffFile is not provided, we just compare the outputs
+    ProcessBuilder diffProcessBuilder;
+    if (diffFile == null) {
+      diffProcessBuilder = createPythonProcessBuilder(DIFF_SCRIPT, pythonOutput, javaOutput);
+    } else {
+      diffProcessBuilder =
+          createPythonProcessBuilder(
+              DIFF_SCRIPT, pythonOutput, javaOutput, "--golden-diff-file", diffFile);
+    }
+
+    Process diffProcess = diffProcessBuilder.inheritIO().start();
+
+    int diffExitCode = diffProcess.waitFor();
+    assertEquals(diffExitCode, 0, "Outputs differ for " + schemaFile.getFileName());
+  }
+}
diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile.avsc b/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile.avsc
new file mode 100644
index 0000000000000..81f8b0e54b11e
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile.avsc
@@ -0,0 +1,456 @@
+{
+    "type": "record",
+    "name": "CustomerProfile",
+    "namespace": "com.example.customer",
+    "doc": "A complex customer profile schema demonstrating various union types and optional fields",
+    "fields": [
+        {
+            "name": "customerId",
+            "type": {
+                "type": "string",
+                "logicalType": "uuid"
+            },
+            "doc": "Unique identifier for the customer"
+        },
+        {
+            "name": "identificationDocument",
+            "type": [
+                "null",
+                {
+                    "type": "record",
+                    "name": "Passport",
+                    "fields": [
+                        {
+                            "name": "passportNumber",
+                            "type": "string"
+                        },
+                        {
+                            "name": "expiryDate",
+                            "type": {
+                                "type": "long",
+                                "logicalType": "date"
+                            }
+                        }
+                    ]
+                },
+                {
+                    "type": "record",
+                    "name": "DriversLicense",
+                    "fields": [
+                        {
+                            "name": "licenseNumber",
+                            "type": "string"
+                        },
+                        {
+                            "name": "state",
+                            "type": "string"
+                        },
+                        {
+                            "name": "validUntil",
+                            "type": {
+                                "type": "long",
+                                "logicalType": "date"
+                            }
+                        }
+                    ]
+                },
+                {
+                    "type": "record",
+                    "name": "NationalID",
+                    "fields": [
+                        {
+                            "name": "idNumber",
+                            "type": "string"
+                        },
+                        {
+                            "name": "country",
+                            "type": "string"
+                        }
+                    ]
+                }
+            ],
+            "default": null,
+            "doc": "Customer's identification document - can be passport, driver's license, or national ID"
+        },
+        {
+            "name": "contactInfo",
+            "type": {
+                "type": "record",
+                "name": "ContactInformation",
+                "fields": [
+                    {
+                        "name": "primaryContact",
+                        "type": [
+                            {
+                                "type": "record",
+                                "name": "EmailContact",
+                                "fields": [
+                                    {
+                                        "name": "emailAddress",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "isVerified",
+                                        "type": "boolean",
+                                        "default": false
+                                    }
+                                ]
+                            },
+                            {
+                                "type": "record",
+                                "name": "PhoneContact",
+                                "fields": [
+                                    {
+                                        "name": "countryCode",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "number",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "type",
+                                        "type": {
+                                            "type": "enum",
+                                            "name": "PhoneType",
+                                            "symbols": [
+                                                "MOBILE",
+                                                "LANDLINE"
+                                            ]
+                                        }
+                                    }
+                                ]
+                            }
+                        ],
+                        "doc": "Primary contact method - either email or phone"
+                    },
+                    {
+                        "name": "alternativeContacts",
+                        "type": {
+                            "type": "array",
+                            "items": [
+                                "null",
+                                "EmailContact",
+                                "PhoneContact"
+                            ]
+                        },
+                        "default": [],
+                        "doc": "List of alternative contact methods"
+                    }
+                ]
+            }
+        },
+        {
+            "name": "addresses",
+            "type": {
+                "type": "array",
+                "items": {
+                    "type": "record",
+                    "name": "Address",
+                    "fields": [
+                        {
+                            "name": "type",
+                            "type": {
+                                "type": "enum",
+                                "name": "AddressType",
+                                "symbols": [
+                                    "RESIDENTIAL",
+                                    "BUSINESS",
+                                    "SHIPPING"
+                                ]
+                            },
+                            "default": "RESIDENTIAL"
+                        },
+                        {
+                            "name": "street",
+                            "type": "string"
+                        },
+                        {
+                            "name": "city",
+                            "type": "string"
+                        },
+                        {
+                            "name": "state",
+                            "type": [
+                                "null",
+                                "string"
+                            ],
+                            "default": null
+                        },
+                        {
+                            "name": "country",
+                            "type": "string"
+                        },
+                        {
+                            "name": "postalCode",
+                            "type": [
+                                "null",
+                                "string"
+                            ],
+                            "default": null
+                        },
+                        {
+                            "name": "validationStatus",
+                            "type": [
+                                "null",
+                                {
+                                    "type": "record",
+                                    "name": "AddressValidation",
+                                    "fields": [
+                                        {
+                                            "name": "isValid",
+                                            "type": "boolean"
+                                        },
+                                        {
+                                            "name": "verificationDate",
+                                            "type": {
+                                                "type": "long",
+                                                "logicalType": "timestamp-millis"
+                                            }
+                                        },
+                                        {
+                                            "name": "verificationMethod",
+                                            "type": {
+                                                "type": "enum",
+                                                "name": "VerificationMethod",
+                                                "symbols": [
+                                                    "MANUAL",
+                                                    "AUTOMATED"
+                                                ]
+                                            }
+                                        }
+                                    ]
+                                }
+                            ],
+                            "default": null
+                        }
+                    ]
+                }
+            },
+            "doc": "Customer's addresses with validation information"
+        },
+        {
+            "name": "preferences",
+            "type": {
+                "type": "map",
+                "values": [
+                    "null",
+                    "string",
+                    "boolean",
+                    {
+                        "type": "record",
+                        "name": "FrequencyPreference",
+                        "fields": [
+                            {
+                                "name": "frequency",
+                                "type": {
+                                    "type": "enum",
+                                    "name": "Frequency",
+                                    "symbols": [
+                                        "DAILY",
+                                        "WEEKLY",
+                                        "MONTHLY"
+                                    ]
+                                }
+                            },
+                            {
+                                "name": "enabled",
+                                "type": "boolean",
+                                "default": true
+                            },
+                            {
+                                "name": "lastUpdated",
+                                "type": {
+                                    "type": "long",
+                                    "logicalType": "timestamp-millis"
+                                }
+                            }
+                        ]
+                    }
+                ]
+            },
+            "doc": "Customer preferences with various possible value types"
+        },
+        {
+            "name": "subscriptionHistory",
+            "type": [
+                "null",
+                {
+                    "type": "array",
+                    "items": {
+                        "type": "record",
+                        "name": "Subscription",
+                        "fields": [
+                            {
+                                "name": "planName",
+                                "type": "string"
+                            },
+                            {
+                                "name": "startDate",
+                                "type": {
+                                    "type": "long",
+                                    "logicalType": "date"
+                                }
+                            },
+                            {
+                                "name": "endDate",
+                                "type": [
+                                    "null",
+                                    {
+                                        "type": "long",
+                                        "logicalType": "date"
+                                    }
+                                ],
+                                "default": null
+                            },
+                            {
+                                "name": "status",
+                                "type": {
+                                    "type": "enum",
+                                    "name": "SubscriptionStatus",
+                                    "symbols": [
+                                        "ACTIVE",
+                                        "CANCELLED",
+                                        "EXPIRED",
+                                        "SUSPENDED"
+                                    ]
+                                }
+                            },
+                            {
+                                "name": "paymentMethod",
+                                "type": [
+                                    "null",
+                                    {
+                                        "type": "record",
+                                        "name": "PaymentMethod",
+                                        "fields": [
+                                            {
+                                                "name": "type",
+                                                "type": {
+                                                    "type": "enum",
+                                                    "name": "PaymentType",
+                                                    "symbols": [
+                                                        "CREDIT_CARD",
+                                                        "DEBIT_CARD",
+                                                        "BANK_TRANSFER",
+                                                        "DIGITAL_WALLET"
+                                                    ]
+                                                }
+                                            },
+                                            {
+                                                "name": "lastFourDigits",
+                                                "type": [
+                                                    "null",
+                                                    "string"
+                                                ],
+                                                "default": null
+                                            },
+                                            {
+                                                "name": "expiryDate",
+                                                "type": [
+                                                    "null",
+                                                    {
+                                                        "type": "long",
+                                                        "logicalType": "date"
+                                                    }
+                                                ],
+                                                "default": null
+                                            }
+                                        ]
+                                    }
+                                ],
+                                "default": null
+                            }
+                        ]
+                    }
+                }
+            ],
+            "default": null,
+            "doc": "Historical record of customer subscriptions"
+        },
+        {
+            "name": "metadata",
+            "type": {
+                "type": "map",
+                "values": [
+                    "null",
+                    "string",
+                    "long",
+                    "boolean",
+                    {
+                        "type": "record",
+                        "name": "MetadataValue",
+                        "fields": [
+                            {
+                                "name": "value",
+                                "type": [
+                                    "null",
+                                    "string",
+                                    "long",
+                                    "boolean"
+                                ],
+                                "default": null
+                            },
+                            {
+                                "name": "timestamp",
+                                "type": {
+                                    "type": "long",
+                                    "logicalType": "timestamp-millis"
+                                }
+                            },
+                            {
+                                "name": "source",
+                                "type": "string"
+                            }
+                        ]
+                    }
+                ]
+            },
+            "doc": "Flexible metadata storage with various possible value types"
+        },
+        {
+            "name": "tags",
+            "type": [
+                "null",
+                {
+                    "type": "array",
+                    "items": {
+                        "type": "record",
+                        "name": "Tag",
+                        "fields": [
+                            {
+                                "name": "name",
+                                "type": "string"
+                            },
+                            {
+                                "name": "value",
+                                "type": [
+                                    "null",
+                                    "string"
+                                ],
+                                "default": null
+                            },
+                            {
+                                "name": "score",
+                                "type": [
+                                    "null",
+                                    "double"
+                                ],
+                                "default": null
+                            },
+                            {
+                                "name": "addedAt",
+                                "type": {
+                                    "type": "long",
+                                    "logicalType": "timestamp-millis"
+                                }
+                            }
+                        ]
+                    }
+                }
+            ],
+            "default": null,
+            "doc": "Optional tags associated with the customer profile"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile2.avsc b/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile2.avsc
new file mode 100644
index 0000000000000..b8c7654ea072a
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile2.avsc
@@ -0,0 +1,244 @@
+{
+    "type": "record",
+    "name": "CustomerProfile2",
+    "namespace": "com.example.customer",
+    "doc": "A complex customer profile schema demonstrating various union types and optional fields",
+    "fields": [
+        {
+            "name": "customerId",
+            "type": {
+                "type": "string",
+                "logicalType": "uuid"
+            },
+            "doc": "Unique identifier for the customer"
+        },
+        {
+            "name": "identificationDocument",
+            "type": [
+                "null",
+                {
+                    "type": "record",
+                    "name": "Passport",
+                    "fields": [
+                        {
+                            "name": "passportNumber",
+                            "type": "string"
+                        },
+                        {
+                            "name": "expiryDate",
+                            "type": {
+                                "type": "long",
+                                "logicalType": "date"
+                            }
+                        }
+                    ]
+                },
+                {
+                    "type": "record",
+                    "name": "DriversLicense",
+                    "fields": [
+                        {
+                            "name": "licenseNumber",
+                            "type": "string"
+                        },
+                        {
+                            "name": "state",
+                            "type": "string"
+                        },
+                        {
+                            "name": "validUntil",
+                            "type": {
+                                "type": "long",
+                                "logicalType": "date"
+                            }
+                        }
+                    ]
+                },
+                {
+                    "type": "record",
+                    "name": "NationalID",
+                    "fields": [
+                        {
+                            "name": "idNumber",
+                            "type": "string"
+                        },
+                        {
+                            "name": "country",
+                            "type": "string"
+                        }
+                    ]
+                }
+            ],
+            "default": null,
+            "doc": "Customer's identification document"
+        },
+        {
+            "name": "contactInfo",
+            "type": {
+                "type": "record",
+                "name": "ContactInformation",
+                "fields": [
+                    {
+                        "name": "primaryEmailContact",
+                        "type": [
+                            "null",
+                            {
+                                "type": "record",
+                                "name": "PrimaryEmailContact",
+                                "fields": [
+                                    {
+                                        "name": "emailAddress",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "isVerified",
+                                        "type": "boolean",
+                                        "default": false
+                                    }
+                                ]
+                            }
+                        ],
+                        "default": null
+                    },
+                    {
+                        "name": "primaryPhoneContact",
+                        "type": [
+                            "null",
+                            {
+                                "type": "record",
+                                "name": "PrimaryPhoneContact",
+                                "fields": [
+                                    {
+                                        "name": "countryCode",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "number",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "type",
+                                        "type": {
+                                            "type": "enum",
+                                            "name": "PhoneType",
+                                            "symbols": [
+                                                "MOBILE",
+                                                "LANDLINE"
+                                            ]
+                                        }
+                                    }
+                                ]
+                            }
+                        ],
+                        "default": null
+                    },
+                    {
+                        "name": "alternativeEmailContacts",
+                        "type": {
+                            "type": "array",
+                            "items": {
+                                "type": "record",
+                                "name": "AlternativeEmailContact",
+                                "fields": [
+                                    {
+                                        "name": "emailAddress",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "isVerified",
+                                        "type": "boolean",
+                                        "default": false
+                                    }
+                                ]
+                            }
+                        },
+                        "default": []
+                    },
+                    {
+                        "name": "alternativePhoneContacts",
+                        "type": {
+                            "type": "array",
+                            "items": {
+                                "type": "record",
+                                "name": "AlternativePhoneContact",
+                                "fields": [
+                                    {
+                                        "name": "countryCode",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "number",
+                                        "type": "string"
+                                    },
+                                    {
+                                        "name": "type",
+                                        "type": "PhoneType"
+                                    }
+                                ]
+                            }
+                        },
+                        "default": []
+                    }
+                ]
+            }
+        },
+        {
+            "name": "preferences",
+            "type": {
+                "type": "record",
+                "name": "Preferences",
+                "fields": [
+                    {
+                        "name": "simplePreferences",
+                        "type": {
+                            "type": "map",
+                            "values": [
+                                "null",
+                                "string",
+                                "boolean"
+                            ]
+                        },
+                        "default": {}
+                    },
+                    {
+                        "name": "frequencyPreferences",
+                        "type": {
+                            "type": "map",
+                            "values": {
+                                "type": "record",
+                                "name": "FrequencyPreference",
+                                "fields": [
+                                    {
+                                        "name": "frequency",
+                                        "type": {
+                                            "type": "enum",
+                                            "name": "Frequency",
+                                            "symbols": [
+                                                "DAILY",
+                                                "WEEKLY",
+                                                "MONTHLY"
+                                            ]
+                                        }
+                                    },
+                                    {
+                                        "name": "enabled",
+                                        "type": "boolean",
+                                        "default": true
+                                    },
+                                    {
+                                        "name": "lastUpdated",
+                                        "type": {
+                                            "type": "long",
+                                            "logicalType": "timestamp-millis"
+                                        }
+                                    }
+                                ]
+                            }
+                        },
+                        "default": {}
+                    }
+                ]
+            }
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/FlatUser.avsc b/metadata-integration/java/datahub-schematron/cli/src/test/resources/FlatUser.avsc
new file mode 100644
index 0000000000000..c796878c32ae4
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/FlatUser.avsc
@@ -0,0 +1,45 @@
+{
+    "type": "record",
+    "name": "FlatUser",
+    "namespace": "com.example",
+    "fields": [
+        {
+            "name": "id",
+            "type": "int",
+            "doc": "The unique identifier for a user",
+            "default": -1,
+            "metadata": {
+                "key1": "value1",
+                "key2": "value2"
+            }
+        },
+        {
+            "name": "username",
+            "type": "string",
+            "doc": "The username of the user"
+        },
+        {
+            "name": "email",
+            "type": "string",
+            "doc": "The email of the user"
+        },
+        {
+            "name": "age",
+            "type": "int",
+            "doc": "The age of the user"
+        },
+        {
+            "name": "isActive",
+            "type": "boolean",
+            "doc": "Whether the user is active or not"
+        },
+        {
+            "name": "registrationDate",
+            "type": {
+                "type": "long",
+                "logicalType": "timestamp-millis"
+            },
+            "doc": "The registration date of the user"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile2_diff.json b/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile2_diff.json
new file mode 100644
index 0000000000000..d4677d722a0cb
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile2_diff.json
@@ -0,0 +1,125 @@
+{
+  "urn:li:dataset:(urn:li:dataPlatform:datahub,com.example.customer.CustomerProfile2,PROD)": {
+    "schemaMetadata": {
+      "added": {},
+      "modified": [
+        "fields"
+      ],
+      "modified_details": {
+        "fields": {
+          "added": {},
+          "modified": [
+            "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=PrimaryEmailContact].primaryEmailContact.[type=boolean].isVerified",
+            "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts",
+            "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts.[type=boolean].isVerified",
+            "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativePhoneContact].alternativePhoneContacts",
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences",
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences.[type=boolean].enabled",
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=boolean].simplePreferences",
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=string].simplePreferences",
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].simplePreferences",
+            "[version=2.0].[type=CustomerProfile2].[type=union].[type=DriversLicense].identificationDocument",
+            "[version=2.0].[type=CustomerProfile2].[type=union].[type=NationalID].identificationDocument",
+            "[version=2.0].[type=CustomerProfile2].[type=union].[type=Passport].identificationDocument",
+            "[version=2.0].[type=CustomerProfile2].[type=union].identificationDocument"
+          ],
+          "modified_details": {
+            "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=PrimaryEmailContact].primaryEmailContact.[type=boolean].isVerified": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: False",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=PrimaryEmailContact].primaryEmailContact.[type=boolean].isVerified"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: []",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts.[type=boolean].isVerified": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: False",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts.[type=boolean].isVerified"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativePhoneContact].alternativePhoneContacts": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: []",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativePhoneContact].alternativePhoneContacts"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: {}",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences.[type=boolean].enabled": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: True",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences.[type=boolean].enabled"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=boolean].simplePreferences": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: {}",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=boolean].simplePreferences"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=string].simplePreferences": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: {}",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=string].simplePreferences"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].simplePreferences": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: {}",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].simplePreferences"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=union].[type=DriversLicense].identificationDocument": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=union].[type=DriversLicense].identificationDocument"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=union].[type=NationalID].identificationDocument": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=union].[type=NationalID].identificationDocument"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=union].[type=Passport].identificationDocument": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=union].[type=Passport].identificationDocument"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile2].[type=union].identificationDocument": {
+              "description": {
+                "after": "Customer's identification document\nField default value: null",
+                "before": "Customer's identification document",
+                "identifier": "[version=2.0].[type=CustomerProfile2].[type=union].identificationDocument"
+              }
+            }
+          },
+          "removed": {}
+        }
+      },
+      "removed": {}
+    }
+  }
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile_diff.json b/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile_diff.json
new file mode 100644
index 0000000000000..4bf0e1074d9a4
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile_diff.json
@@ -0,0 +1,181 @@
+{
+  "urn:li:dataset:(urn:li:dataPlatform:datahub,com.example.customer.CustomerProfile,PROD)": {
+    "schemaMetadata": {
+      "added": {},
+      "modified": [
+        "fields"
+      ],
+      "modified_details": {
+        "fields": {
+          "added": {
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts.[type=boolean].isVerified": {
+              "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts.[type=boolean].isVerified",
+              "isPartOfKey": false,
+              "nativeDataType": "boolean",
+              "nullable": false,
+              "type": {
+                "type": {
+                  "com.linkedin.schema.BooleanType": {}
+                }
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts.[type=string].emailAddress": {
+              "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts.[type=string].emailAddress",
+              "isPartOfKey": false,
+              "nativeDataType": "string",
+              "nullable": false,
+              "type": {
+                "type": {
+                  "com.linkedin.schema.StringType": {}
+                }
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=enum].type": {
+              "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=enum].type",
+              "isPartOfKey": false,
+              "nativeDataType": "Enum",
+              "nullable": false,
+              "type": {
+                "type": {
+                  "com.linkedin.schema.EnumType": {}
+                }
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=string].countryCode": {
+              "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=string].countryCode",
+              "isPartOfKey": false,
+              "nativeDataType": "string",
+              "nullable": false,
+              "type": {
+                "type": {
+                  "com.linkedin.schema.StringType": {}
+                }
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=string].number": {
+              "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=string].number",
+              "isPartOfKey": false,
+              "nativeDataType": "string",
+              "nullable": false,
+              "type": {
+                "type": {
+                  "com.linkedin.schema.StringType": {}
+                }
+              }
+            }
+          },
+          "modified": [
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts",
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts",
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].alternativeContacts",
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=union].[type=EmailContact].primaryContact.[type=boolean].isVerified",
+            "[version=2.0].[type=CustomerProfile].[type=array].[type=Address].addresses.[type=enum].type",
+            "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=FrequencyPreference].preferences.[type=boolean].enabled",
+            "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=boolean].value",
+            "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=long].value",
+            "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=string].value",
+            "[version=2.0].[type=CustomerProfile].[type=union].[type=DriversLicense].identificationDocument",
+            "[version=2.0].[type=CustomerProfile].[type=union].[type=NationalID].identificationDocument",
+            "[version=2.0].[type=CustomerProfile].[type=union].[type=Passport].identificationDocument",
+            "[version=2.0].[type=CustomerProfile].[type=union].identificationDocument"
+          ],
+          "modified_details": {
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts": {
+              "description": {
+                "after": "List of alternative contact methods",
+                "before": "List of alternative contact methods\nField default value: []",
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts": {
+              "description": {
+                "after": "List of alternative contact methods",
+                "before": "List of alternative contact methods\nField default value: []",
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].alternativeContacts": {
+              "description": {
+                "after": "List of alternative contact methods",
+                "before": "List of alternative contact methods\nField default value: []",
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].alternativeContacts"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=union].[type=EmailContact].primaryContact.[type=boolean].isVerified": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: False",
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=union].[type=EmailContact].primaryContact.[type=boolean].isVerified"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=array].[type=Address].addresses.[type=enum].type": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: RESIDENTIAL",
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=array].[type=Address].addresses.[type=enum].type"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=FrequencyPreference].preferences.[type=boolean].enabled": {
+              "description": {
+                "after": null,
+                "before": "\nField default value: True",
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=FrequencyPreference].preferences.[type=boolean].enabled"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=boolean].value": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=boolean].value"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=long].value": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=long].value"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=string].value": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=string].value"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=union].[type=DriversLicense].identificationDocument": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=union].[type=DriversLicense].identificationDocument"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=union].[type=NationalID].identificationDocument": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=union].[type=NationalID].identificationDocument"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=union].[type=Passport].identificationDocument": {
+              "nullable": {
+                "after": false,
+                "before": true,
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=union].[type=Passport].identificationDocument"
+              }
+            },
+            "[version=2.0].[type=CustomerProfile].[type=union].identificationDocument": {
+              "description": {
+                "after": "Customer's identification document - can be passport, driver's license, or national ID\nField default value: null",
+                "before": "Customer's identification document - can be passport, driver's license, or national ID",
+                "identifier": "[version=2.0].[type=CustomerProfile].[type=union].identificationDocument"
+              }
+            }
+          },
+          "removed": {}
+        }
+      },
+      "removed": {}
+    }
+  }
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/lib/build.gradle b/metadata-integration/java/datahub-schematron/lib/build.gradle
new file mode 100644
index 0000000000000..3ba22ff4cb7b5
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/build.gradle
@@ -0,0 +1,126 @@
+plugins {
+    id("com.palantir.git-version") apply false
+}
+apply plugin: 'java'
+apply plugin: 'jacoco'
+apply plugin: 'signing'
+apply plugin: 'io.codearte.nexus-staging'
+apply plugin: 'maven-publish'
+apply from: '../../versioning.gradle'
+
+dependencies {
+
+    implementation project(':entity-registry')
+//
+//    // Jackson dependencies - use the same versions as in the parent project
+//    implementation 'com.fasterxml.jackson.core:jackson-core:2.12.3'
+//    implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3'
+//    implementation 'com.fasterxml.jackson.core:jackson-annotations:2.12.3'
+    
+    // Core dependencies
+//    implementation externalDependency.guava
+//    implementation externalDependency.gson
+//    implementation externalDependency.commonsCli
+//    implementation externalDependency.slf4jApi
+//    implementation externalDependency.jacksonCore
+    
+    // Schema format dependencies
+//    implementation externalDependency.protobuf
+    implementation externalDependency.avro
+//    implementation 'org.apache.thrift:libthrift:0.16.0'
+//    implementation 'io.swagger.parser.v3:swagger-parser:2.1.12'
+    
+    // Utilities
+    compileOnly externalDependency.lombok
+    annotationProcessor externalDependency.lombok
+    
+    // Testing
+    testImplementation externalDependency.testng
+    testImplementation 'org.mockito:mockito-core:5.3.1'
+}
+
+jacocoTestReport {
+    dependsOn test
+}
+
+test.finalizedBy jacocoTestReport
+
+configurations {
+    provided
+    implementation.extendsFrom provided
+}
+
+java {
+    withJavadocJar()
+    withSourcesJar()
+}
+
+publishing {
+    publications {
+        mavenJava(MavenPublication) {
+            from components.java
+
+            pom {
+                name = 'Datahub Schematron'
+                groupId = 'io.acryl'
+                artifactId = 'datahub-schematron'
+                description = 'DataHub schema translation library for converting between different schema formats using DataHub as an intermediate representation'
+                url = 'https://datahubproject.io'
+
+                scm {
+                    connection = 'scm:git:git://github.com/datahub-project/datahub.git'
+                    developerConnection = 'scm:git:ssh://github.com:datahub-project/datahub.git'
+                    url = 'https://github.com/datahub-project/datahub.git'
+                }
+
+                licenses {
+                    license {
+                        name = 'The Apache License, Version 2.0'
+                        url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+                    }
+                }
+
+                developers {
+                    developer {
+                        id = 'datahub'
+                        name = 'Datahub'
+                        email = 'datahub@acryl.io'
+                    }
+                }
+            }
+        }
+    }
+
+    repositories {
+        maven {
+            def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/"
+            def snapshotsRepoUrl = "https://s01.oss.sonatype.org/content/repositories/snapshots/"
+            def ossrhUsername = System.getenv('RELEASE_USERNAME')
+            def ossrhPassword = System.getenv('RELEASE_PASSWORD')
+            credentials {
+                username ossrhUsername
+                password ossrhPassword
+            }
+            url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
+        }
+    }
+}
+
+signing {
+    def signingKey = findProperty("signingKey")
+    def signingPassword = System.getenv("SIGNING_PASSWORD")
+    // Only require signing if we have the signing key property
+    required = signingKey != null
+
+    if (signingKey != null) {
+        useInMemoryPgpKeys(signingKey, signingPassword)
+        sign publishing.publications.mavenJava
+    }
+
+}
+
+nexusStaging {
+    serverUrl = "https://s01.oss.sonatype.org/service/local/"
+    username = System.getenv("NEXUS_USERNAME")
+    password = System.getenv("NEXUS_PASSWORD")
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/SchemaConverter.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/SchemaConverter.java
new file mode 100644
index 0000000000000..cb364f2c7a1a2
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/SchemaConverter.java
@@ -0,0 +1,25 @@
+package io.datahubproject.schematron.converters;
+
+import com.linkedin.common.urn.DataPlatformUrn;
+import com.linkedin.schema.SchemaMetadata;
+
+/** Base interface for converting between different schema formats. */
+public interface SchemaConverter<T> {
+  /**
+   * Converts a schema into DataHub's SchemaField format.
+   *
+   * @param schema The source schema to convert
+   * @param isKeySchema Whether this represents a key schema
+   * @param defaultNullable Default nullable setting for fields
+   * @param platformUrn Data platform urn
+   * @param rawSchemaString Raw schema string (if available). When provided - it will be used to
+   *     generate the schema fingerprint
+   * @return List of SchemaFields representing the schema in DataHub's format
+   */
+  SchemaMetadata toDataHubSchema(
+      T schema,
+      boolean isKeySchema,
+      boolean defaultNullable,
+      DataPlatformUrn platformUrn,
+      String rawSchemaString);
+}
diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java
new file mode 100644
index 0000000000000..0ddb357db76ba
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java
@@ -0,0 +1,611 @@
+package io.datahubproject.schematron.converters.avro;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.linkedin.common.urn.DataPlatformUrn;
+import com.linkedin.schema.*;
+import io.datahubproject.schematron.converters.SchemaConverter;
+import io.datahubproject.schematron.models.*;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.function.Supplier;
+import lombok.Builder;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.avro.JsonProperties;
+import org.apache.avro.LogicalType;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaNormalization;
+
+/** Converts Avro schemas to DataHub's schema format following SchemaFieldPath Specification V2. */
+@Slf4j
+@Builder
+public class AvroSchemaConverter implements SchemaConverter<Schema> {
+
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+  private static final Map<String, Supplier<SchemaFieldDataType.Type>> LOGICAL_TYPE_MAPPING;
+
+  static {
+    Map<String, Supplier<SchemaFieldDataType.Type>> logicalTypeMap = new HashMap<>();
+    logicalTypeMap.put("date", () -> SchemaFieldDataType.Type.create(new DateType()));
+    logicalTypeMap.put("time-micros", () -> SchemaFieldDataType.Type.create(new TimeType()));
+    logicalTypeMap.put("time-millis", () -> SchemaFieldDataType.Type.create(new TimeType()));
+    logicalTypeMap.put("timestamp-micros", () -> SchemaFieldDataType.Type.create(new TimeType()));
+    logicalTypeMap.put("timestamp-millis", () -> SchemaFieldDataType.Type.create(new TimeType()));
+    logicalTypeMap.put("decimal", () -> SchemaFieldDataType.Type.create(new NumberType()));
+    logicalTypeMap.put("uuid", () -> SchemaFieldDataType.Type.create(new StringType()));
+    LOGICAL_TYPE_MAPPING = Collections.unmodifiableMap(logicalTypeMap);
+  }
+
+  private SchemaFieldDataType.Type getTypeFromLogicalType(Schema schema) {
+    LogicalType logicalType = schema.getLogicalType();
+    if (logicalType != null) {
+      Supplier<SchemaFieldDataType.Type> typeSupplier =
+          LOGICAL_TYPE_MAPPING.get(logicalType.getName());
+      if (typeSupplier != null) {
+        return typeSupplier.get();
+      }
+    }
+    return getBaseType(schema);
+  }
+
+  private SchemaFieldDataType.Type getBaseType(Schema schema) {
+    switch (schema.getType()) {
+      case BOOLEAN:
+        return SchemaFieldDataType.Type.create(new BooleanType());
+      case INT:
+      case LONG:
+      case FLOAT:
+      case DOUBLE:
+        return SchemaFieldDataType.Type.create(new NumberType());
+      case STRING:
+        return SchemaFieldDataType.Type.create(new StringType());
+      case BYTES:
+        return SchemaFieldDataType.Type.create(new BytesType());
+      case FIXED:
+        return SchemaFieldDataType.Type.create(new FixedType());
+      case ENUM:
+        return SchemaFieldDataType.Type.create(new EnumType());
+      case ARRAY:
+        return SchemaFieldDataType.Type.create(new ArrayType());
+      case MAP:
+        return SchemaFieldDataType.Type.create(new MapType());
+      case RECORD:
+        return SchemaFieldDataType.Type.create(new RecordType());
+      case UNION:
+        return SchemaFieldDataType.Type.create(new UnionType());
+      default:
+        return SchemaFieldDataType.Type.create(new NullType());
+    }
+  }
+
+  private String getFieldType(Schema schema) {
+    // For the field path, we just want the base type without the logical type
+    return schema.getType().getName().toLowerCase();
+  }
+
+  private String getNativeDataType(Schema schema) {
+    // For native data type, we can include the logical type information
+    LogicalType logicalType = schema.getLogicalType();
+    if (logicalType != null) {
+      return schema.getType().getName().toLowerCase() + "(" + logicalType.getName() + ")";
+    }
+    return schema.getType().getName().toLowerCase();
+  }
+
+  @Override
+  public SchemaMetadata toDataHubSchema(
+      Schema schema,
+      boolean isKeySchema,
+      boolean defaultNullable,
+      DataPlatformUrn platformUrn,
+      String rawSchemaString) {
+
+    try {
+      byte[] fingerprintBytes = null;
+      try {
+        if (rawSchemaString != null) {
+          String canonicalForm = SchemaNormalization.toParsingForm(schema);
+          log.debug("Length of canonical form: {}", canonicalForm.length());
+          log.debug("Canonical form: {}", canonicalForm);
+          fingerprintBytes =
+              SchemaNormalization.fingerprint(
+                  "MD5", rawSchemaString.getBytes(StandardCharsets.UTF_8));
+        } else {
+          fingerprintBytes = SchemaNormalization.parsingFingerprint("MD5", schema);
+        }
+      } catch (Exception e) {
+        log.error("Failed to compute schema fingerprint", e);
+      }
+
+      String schemaHash = "";
+      if (fingerprintBytes != null) {
+        // Convert to hex string
+        StringBuilder hexString = new StringBuilder();
+        for (byte b : fingerprintBytes) {
+          hexString.append(String.format("%02x", b));
+        }
+        schemaHash = hexString.toString();
+      }
+
+      List<SchemaField> fields = new ArrayList<>();
+      FieldPath basePath = new FieldPath();
+      basePath.setKeySchema(isKeySchema);
+
+      // Add the record type to the base path
+      if (schema.getType() == Schema.Type.RECORD) {
+        basePath = basePath.expandType(schema.getName(), schema.toString());
+      }
+
+      processSchema(schema, basePath, defaultNullable, fields);
+
+      return new SchemaMetadata()
+          .setSchemaName(schema.getName())
+          .setPlatform(platformUrn)
+          .setVersion(0)
+          .setHash(schemaHash)
+          .setPlatformSchema(
+              SchemaMetadata.PlatformSchema.create(
+                  new OtherSchema().setRawSchema(schema.toString())))
+          .setFields(new SchemaFieldArray(fields));
+
+    } catch (Exception e) {
+      log.error("Failed to convert Avro schema", e);
+      throw new RuntimeException("Failed to convert Avro schema", e);
+    }
+  }
+
+  private void processSchema(
+      Schema schema, FieldPath fieldPath, boolean defaultNullable, List<SchemaField> fields) {
+    if (schema.getType() == Schema.Type.RECORD) {
+      for (Schema.Field field : schema.getFields()) {
+        processField(field, fieldPath, defaultNullable, fields);
+      }
+    }
+  }
+
+  private void processField(
+      Schema.Field field, FieldPath fieldPath, boolean defaultNullable, List<SchemaField> fields) {
+    processField(field, fieldPath, defaultNullable, fields, false, null);
+  }
+
+  private void processField(
+      Schema.Field field,
+      FieldPath fieldPath,
+      boolean defaultNullable,
+      List<SchemaField> fields,
+      boolean nullableOverride) {
+    processField(field, fieldPath, defaultNullable, fields, nullableOverride, null);
+  }
+
+  private void processField(
+      Schema.Field field,
+      FieldPath fieldPath,
+      boolean defaultNullable,
+      List<SchemaField> fields,
+      boolean nullableOverride,
+      DataHubType typeOverride) {
+    log.debug(
+        "Processing field: {}, Field path : {}, Field schema: {}",
+        field.name(),
+        fieldPath.asString(),
+        field.schema());
+    Schema fieldSchema = field.schema();
+    boolean isNullable = isNullable(fieldSchema, defaultNullable);
+    if (nullableOverride) {
+      // If a nullable override is provided, use the override value
+      isNullable = true;
+    }
+    if (typeOverride != null) {
+      // If a type override is provided, use the nullable value from the override
+      isNullable = nullableOverride;
+    }
+    log.debug(
+        "DefaultNullability: {}, Determined nullability for field name: {} at path: {} is {}",
+        defaultNullable,
+        field.name(),
+        fieldPath.asString(),
+        isNullable);
+    String discriminatedType = getDiscriminatedType(fieldSchema);
+
+    FieldElement element =
+        new FieldElement(new ArrayList<>(), new ArrayList<>(), field.name(), typeOverride);
+
+    FieldPath newPath = fieldPath.clonePlus(element);
+
+    switch (fieldSchema.getType()) {
+      case RECORD:
+        processRecordField(
+            field, newPath, discriminatedType, defaultNullable, fields, isNullable, typeOverride);
+        break;
+      case ARRAY:
+        processArrayField(field, newPath, discriminatedType, defaultNullable, fields, isNullable);
+        break;
+      case MAP:
+        processMapField(field, newPath, discriminatedType, defaultNullable, fields, isNullable);
+        break;
+      case UNION:
+        processUnionField(
+            field, newPath, discriminatedType, defaultNullable, fields, isNullable, typeOverride);
+        break;
+      case ENUM:
+        processEnumField(field, newPath, discriminatedType, defaultNullable, fields, isNullable);
+        break;
+      default:
+        processPrimitiveField(
+            field, newPath, discriminatedType, defaultNullable, fields, isNullable);
+        break;
+    }
+  }
+
+  private void processRecordField(
+      Schema.Field field,
+      FieldPath fieldPath,
+      String discriminatedType,
+      boolean defaultNullable,
+      List<SchemaField> fields,
+      boolean isNullable,
+      DataHubType typeOverride) {
+
+    log.debug("Record Field Path before expand: {}", fieldPath.asString());
+    FieldPath recordPath = fieldPath.expandType(discriminatedType, field.schema().toString());
+    log.debug("Record Field Path after expand: {}", recordPath.asString());
+
+    SchemaFieldDataType dataType =
+        typeOverride != null
+            ? typeOverride.asSchemaFieldType()
+            : new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType()));
+
+    // Add the record field itself
+    SchemaField recordField =
+        new SchemaField()
+            .setFieldPath(recordPath.asString())
+            .setType(dataType)
+            .setNativeDataType(discriminatedType)
+            .setNullable(isNullable || defaultNullable)
+            .setIsPartOfKey(fieldPath.isKeySchema());
+
+    populateCommonProperties(field, recordField);
+
+    fields.add(recordField);
+
+    // Process nested fields
+    for (Schema.Field nestedField : field.schema().getFields()) {
+      processField(nestedField, recordPath, defaultNullable, fields);
+    }
+  }
+
+  @SneakyThrows
+  private static void populateCommonProperties(Schema.Field field, SchemaField datahubField) {
+    // Create a new mutable HashMap to store combined properties
+    Map<String, Object> combinedProps = new HashMap<>();
+
+    // Add properties from field if any exist
+    Map<String, Object> fieldProps = field.getObjectProps();
+    if (fieldProps != null) {
+      combinedProps.putAll(fieldProps);
+    }
+
+    // Add properties from schema if any exist
+    Map<String, Object> schemaProps = field.schema().getObjectProps();
+    if (schemaProps != null) {
+      combinedProps.putAll(schemaProps);
+    }
+
+    // Only proceed with serialization if we have properties
+    if (!combinedProps.isEmpty()) {
+      try {
+        String jsonSerializedProps = OBJECT_MAPPER.writeValueAsString(combinedProps);
+        datahubField.setJsonProps(jsonSerializedProps);
+      } catch (Exception e) {
+        log.error(
+            "Non-fatal error. Failed to serialize schema properties for field: " + field.name(), e);
+      }
+    }
+
+    // Set the description if it exists
+    if (field.doc() != null && !field.doc().isEmpty()) {
+      datahubField.setDescription(field.doc());
+      if (field.hasDefaultValue()) {
+        Object defaultValue = field.defaultVal();
+        // if the default value is the JSON NULL node, then we handle it differently
+        if (defaultValue == JsonProperties.NULL_VALUE) {
+          datahubField.setDescription(
+              datahubField.getDescription() + "\nField default value: null");
+        } else {
+          datahubField.setDescription(
+              datahubField.getDescription()
+                  + "\nField default value: "
+                  + OBJECT_MAPPER.writeValueAsString(defaultValue));
+        }
+      }
+    }
+  }
+
+  private void processArrayField(
+      Schema.Field field,
+      FieldPath fieldPath,
+      String discriminatedType,
+      boolean defaultNullable,
+      List<SchemaField> fields,
+      boolean isNullable) {
+
+    Schema arraySchema = field.schema();
+    Schema elementSchema = arraySchema.getElementType();
+    String elementType = getDiscriminatedType(elementSchema);
+
+    fieldPath = fieldPath.expandType("array", arraySchema);
+    // Set parent type for proper array handling
+    DataHubType arrayDataHubType = new DataHubType(ArrayType.class, elementType);
+
+    // Process element type if it's complex
+    if (elementSchema.getType() == Schema.Type.RECORD
+        || elementSchema.getType() == Schema.Type.ARRAY
+        || elementSchema.getType() == Schema.Type.MAP
+        || elementSchema.getType() == Schema.Type.UNION) {
+      log.debug("Array Field Path before expand: {}", fieldPath.asString());
+      fieldPath = fieldPath.popLast();
+      fieldPath =
+          fieldPath.clonePlus(
+              new FieldElement(Collections.singletonList("array"), new ArrayList<>(), null, null));
+      Schema.Field elementField =
+          new Schema.Field(
+              field.name(),
+              elementSchema,
+              elementSchema.getDoc() != null ? elementSchema.getDoc() : field.doc(),
+              null // TODO: What is the default value for an array element?
+              );
+      processField(elementField, fieldPath, defaultNullable, fields, isNullable, arrayDataHubType);
+    } else {
+
+      SchemaField arrayField =
+          new SchemaField()
+              .setFieldPath(fieldPath.asString())
+              .setType(arrayDataHubType.asSchemaFieldType())
+              .setNativeDataType("array(" + elementType + ")")
+              .setNullable(isNullable || defaultNullable)
+              .setIsPartOfKey(fieldPath.isKeySchema());
+
+      populateCommonProperties(field, arrayField);
+      log.debug("Array field path: {} with doc: {}", fieldPath.asString(), field.doc());
+      fields.add(arrayField);
+    }
+  }
+
+  private void processMapField(
+      Schema.Field field,
+      FieldPath fieldPath,
+      String discriminatedType,
+      boolean defaultNullable,
+      List<SchemaField> fields,
+      boolean isNullable) {
+
+    Schema mapSchema = field.schema();
+    Schema valueSchema = mapSchema.getValueType();
+    String valueType = getDiscriminatedType(valueSchema);
+
+    DataHubType mapDataHubType = new DataHubType(MapType.class, valueType);
+    fieldPath = fieldPath.expandType("map", mapSchema);
+
+    // Process value type if it's complex
+    if (valueSchema.getType() == Schema.Type.RECORD
+        || valueSchema.getType() == Schema.Type.ARRAY
+        || valueSchema.getType() == Schema.Type.MAP
+        || valueSchema.getType() == Schema.Type.UNION) {
+      Schema.Field valueField =
+          new Schema.Field(
+              field.name(),
+              valueSchema,
+              valueSchema.getDoc() != null ? valueSchema.getDoc() : field.doc(),
+              null // TODO: What is the default value for a map value?
+              ); // Nullability for map values follows the nullability of the map itself
+      FieldPath valueFieldPath =
+          fieldPath
+              .popLast()
+              .clonePlus(
+                  new FieldElement(
+                      Collections.singletonList("map"), new ArrayList<>(), null, null));
+      processField(valueField, valueFieldPath, defaultNullable, fields, isNullable, mapDataHubType);
+    } else {
+      SchemaField mapField =
+          new SchemaField()
+              .setFieldPath(fieldPath.asString())
+              .setType(mapDataHubType.asSchemaFieldType())
+              .setNativeDataType("map<string," + valueType + ">")
+              .setNullable(isNullable || defaultNullable)
+              .setIsPartOfKey(fieldPath.isKeySchema());
+
+      populateCommonProperties(field, mapField);
+      fields.add(mapField);
+    }
+  }
+
+  private void processUnionField(
+      Schema.Field field,
+      FieldPath fieldPath,
+      String discriminatedType,
+      boolean defaultNullable,
+      List<SchemaField> fields,
+      boolean isNullable,
+      DataHubType typeOverride) {
+
+    List<Schema> unionTypes = field.schema().getTypes();
+
+    // If this is just a nullable type (union with null), process the non-null type
+    // directly
+    if (unionTypes.size() == 2 && isNullable) {
+      Schema nonNullSchema =
+          unionTypes.stream()
+              .filter(s -> s.getType() != Schema.Type.NULL)
+              .findFirst()
+              .orElseThrow(NoSuchElementException::new);
+
+      processField(
+          new Schema.Field(field.name(), nonNullSchema, field.doc()),
+          fieldPath.popLast(),
+          defaultNullable,
+          fields,
+          true);
+      return;
+    }
+
+    log.debug("Union Field Path before expand: {}", fieldPath.asString());
+
+    // Otherwise, process as a true union type
+    DataHubType unionDataHubType = new DataHubType(UnionType.class, discriminatedType);
+    FieldPath unionFieldPath = fieldPath.expandType("union", field.schema().toString());
+    log.debug("Union Field Path after expand: {}", unionFieldPath.asString());
+
+    SchemaField unionField =
+        new SchemaField()
+            .setFieldPath(unionFieldPath.asString())
+            .setType(
+                typeOverride == null
+                    ? unionDataHubType.asSchemaFieldType()
+                    : typeOverride.asSchemaFieldType())
+            .setNativeDataType("union")
+            .setNullable(isNullable || defaultNullable)
+            .setIsPartOfKey(fieldPath.isKeySchema());
+
+    populateCommonProperties(field, unionField);
+    fields.add(unionField);
+
+    String unionDescription = field.doc() != null ? field.doc() : field.schema().getDoc();
+
+    // Process each union type
+    int typeIndex = 0;
+    for (Schema unionSchema : unionTypes) {
+      if (unionSchema.getType() != Schema.Type.NULL) {
+        log.debug("TypeIndex: {}, Field path : {}", typeIndex, fieldPath.asString());
+        FieldPath indexedFieldPath = fieldPath.popLast();
+        indexedFieldPath =
+            indexedFieldPath.clonePlus(
+                new FieldElement(
+                    Collections.singletonList("union"), new ArrayList<>(), null, null));
+        log.debug("TypeIndex: {}, Indexed Field path : {}", typeIndex, indexedFieldPath.asString());
+        // FieldPath unionFieldPath =
+        // fieldPath.expandType(getDiscriminatedType(unionSchema),
+        // unionSchema.toString());
+        log.debug("TypeIndex: {}, Union Field path : {}", typeIndex, unionFieldPath.asString());
+        String unionFieldName = field.name();
+        Schema.Field unionFieldInner =
+            new Schema.Field(
+                unionFieldName,
+                unionSchema,
+                unionSchema.getDoc() != null ? unionSchema.getDoc() : unionDescription,
+                null);
+        log.debug(
+            "TypeIndex: {}, Union Field path : {}, Doc: {}",
+            typeIndex,
+            unionFieldPath.asString(),
+            unionFieldInner.doc());
+        processField(unionFieldInner, indexedFieldPath, defaultNullable, fields);
+      }
+      typeIndex++;
+    }
+  }
+
+  private void processEnumField(
+      Schema.Field field,
+      FieldPath fieldPath,
+      String discriminatedType,
+      boolean defaultNullable,
+      List<SchemaField> fields,
+      boolean isNullable) {
+
+    fieldPath = fieldPath.expandType("enum", field.schema().toString());
+
+    String enumDescription = field.doc() != null ? field.doc() : "";
+    enumDescription +=
+        " Allowed symbols are: " + String.join(", ", field.schema().getEnumSymbols());
+
+    SchemaField enumField =
+        new SchemaField()
+            .setFieldPath(fieldPath.asString())
+            .setType(
+                new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new EnumType())))
+            .setNativeDataType("Enum")
+            .setNullable(isNullable || defaultNullable)
+            .setIsPartOfKey(fieldPath.isKeySchema());
+
+    populateCommonProperties(field, enumField);
+
+    if (field.doc() != null && !field.doc().isEmpty()) {
+      enumField.setDescription(enumDescription);
+    }
+
+    fields.add(enumField);
+  }
+
+  @SneakyThrows
+  private void processPrimitiveField(
+      Schema.Field field,
+      FieldPath fieldPath,
+      String discriminatedType,
+      boolean defaultNullable,
+      List<SchemaField> fields,
+      boolean isNullable) {
+
+    fieldPath = fieldPath.expandType(discriminatedType, field.schema().toString());
+    SchemaField primitiveField =
+        new SchemaField()
+            .setFieldPath(fieldPath.asString())
+            .setType(new SchemaFieldDataType().setType(getTypeFromLogicalType(field.schema())))
+            .setNativeDataType(getNativeDataType(field.schema()))
+            .setNullable(isNullable || defaultNullable)
+            .setIsPartOfKey(fieldPath.isKeySchema());
+
+    populateCommonProperties(field, primitiveField);
+
+    fields.add(primitiveField);
+  }
+
+  private boolean isNullable(Schema schema, boolean defaultNullable) {
+    if (schema.getType() == Schema.Type.UNION) {
+      return schema.getTypes().stream().anyMatch(type -> type.getType() == Schema.Type.NULL);
+    }
+    return defaultNullable;
+  }
+
+  /**
+   * for record type we want to include the fully qualified name stripped of the namespace
+   *
+   * @param schema
+   * @return
+   */
+  private String getDiscriminatedType(Schema schema) {
+
+    if (schema.getType() == Schema.Type.RECORD) {
+      if (schema.getNamespace() != null) {
+        return schema.getFullName().substring(schema.getNamespace().length() + 1);
+      } else {
+        return schema.getFullName();
+      }
+    }
+    return schema.getType().getName().toLowerCase();
+  }
+
+  private SchemaFieldDataType getPrimitiveFieldType(Schema schema) {
+
+    SchemaFieldDataType fieldType = new SchemaFieldDataType();
+    switch (schema.getType()) {
+      case BOOLEAN:
+        fieldType.setType(SchemaFieldDataType.Type.create(new BooleanType()));
+        break;
+      case INT:
+      case LONG:
+      case FLOAT:
+      case DOUBLE:
+        fieldType.setType(SchemaFieldDataType.Type.create(new NumberType()));
+        break;
+      case STRING:
+        fieldType.setType(SchemaFieldDataType.Type.create(new StringType()));
+        break;
+      case BYTES:
+        fieldType.setType(SchemaFieldDataType.Type.create(new BytesType()));
+        break;
+      default:
+        fieldType.setType(SchemaFieldDataType.Type.create(new NullType()));
+    }
+    return fieldType;
+  }
+}
diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/DataHubType.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/DataHubType.java
new file mode 100644
index 0000000000000..ec6e8ce5a3554
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/DataHubType.java
@@ -0,0 +1,40 @@
+package io.datahubproject.schematron.models;
+
+import com.linkedin.data.template.StringArray;
+import com.linkedin.schema.*;
+import lombok.Data;
+
+@Data
+public class DataHubType {
+  private Class type;
+  private String nestedType;
+
+  public DataHubType(Class type, String nestedType) {
+    this.type = type;
+    this.nestedType = nestedType;
+  }
+
+  public SchemaFieldDataType asSchemaFieldType() {
+    if (type == UnionType.class) {
+      return new SchemaFieldDataType()
+          .setType(
+              SchemaFieldDataType.Type.create(
+                  new UnionType()
+                      .setNestedTypes(nestedType != null ? new StringArray(nestedType) : null)));
+    } else if (type == ArrayType.class) {
+      return new SchemaFieldDataType()
+          .setType(
+              SchemaFieldDataType.Type.create(
+                  new ArrayType()
+                      .setNestedType(nestedType != null ? new StringArray(nestedType) : null)));
+    } else if (type == MapType.class) {
+      return new SchemaFieldDataType()
+          .setType(
+              SchemaFieldDataType.Type.create(
+                  new MapType()
+                      .setKeyType("string")
+                      .setValueType(nestedType != null ? nestedType : null)));
+    }
+    throw new IllegalArgumentException("Unexpected type " + type);
+  }
+}
diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldElement.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldElement.java
new file mode 100644
index 0000000000000..6cdde845d9561
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldElement.java
@@ -0,0 +1,38 @@
+package io.datahubproject.schematron.models;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import lombok.Data;
+
+@Data
+public class FieldElement {
+  private List<String> type;
+  private List<String> schemaTypes;
+  private String name;
+  private DataHubType parentType;
+
+  public FieldElement(
+      List<String> type, List<String> schemaTypes, String name, DataHubType parentType) {
+    this.type = type;
+    this.schemaTypes = schemaTypes;
+    this.name = name;
+    this.parentType = parentType;
+  }
+
+  public FieldElement clone() {
+    return new FieldElement(new ArrayList<>(type), new ArrayList<>(schemaTypes), name, parentType);
+  }
+
+  public String asString(boolean v2Format) {
+    if (v2Format) {
+      String typePrefix =
+          type.stream()
+              .map(innerType -> "[type=" + innerType + "]")
+              .collect(Collectors.joining("."));
+      return name != null ? typePrefix + "." + name : typePrefix;
+    } else {
+      return name != null ? name : "";
+    }
+  }
+}
diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java
new file mode 100644
index 0000000000000..b4b72fcc031a5
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java
@@ -0,0 +1,174 @@
+package io.datahubproject.schematron.models;
+
+import com.linkedin.schema.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import lombok.Data;
+import lombok.NonNull;
+
+@Data
+public class FieldPath {
+  public static final String EMPTY_FIELD_NAME = " ";
+  @NonNull private List<FieldElement> path;
+  private boolean isKeySchema;
+  private boolean useV2PathsAlways;
+
+  public FieldPath() {
+    this.path = new ArrayList<>();
+    this.isKeySchema = false;
+    this.useV2PathsAlways = true;
+  }
+
+  public void setPath(List<FieldElement> path) {
+    if (path == null) {
+      throw new IllegalArgumentException("Path cannot be null");
+    }
+    // Ensure that no element in the path is null
+    if (path.stream().anyMatch(Objects::isNull)) {
+      throw new IllegalArgumentException("Path cannot contain null elements");
+    }
+    this.path = path;
+  }
+
+  private boolean needsV2Path() {
+    if (useV2PathsAlways) {
+      return true;
+    }
+    if (isKeySchema) {
+      return true;
+    }
+    return path.stream()
+        .flatMap(element -> element.getType().stream())
+        .anyMatch(t -> t.equals("union") || t.equals("array"));
+  }
+
+  private void setParentTypeIfNotExists(DataHubType parentType) {
+    if (!path.isEmpty() && path.get(path.size() - 1).getParentType() == null) {
+      path.get(path.size() - 1).setParentType(parentType);
+    }
+  }
+
+  private SchemaFieldDataType getTypeOverride() {
+    if (!path.isEmpty() && path.get(path.size() - 1).getParentType() != null) {
+      return path.get(path.size() - 1).getParentType().asSchemaFieldType();
+    }
+    return null;
+  }
+
+  private String getNativeTypeOverride() {
+    SchemaFieldDataType typeOverride = getTypeOverride();
+    if (typeOverride != null) {
+      if (typeOverride.getType().isArrayType()) {
+        ArrayType arrayType = typeOverride.getType().getArrayType();
+        return String.format(
+            "array(%s)",
+            arrayType.getNestedType() != null ? String.join(",", arrayType.getNestedType()) : "");
+      } else if (typeOverride.getType().isMapType()) {
+        MapType mapType = typeOverride.getType().getMapType();
+        return String.format("map(str,%s)", mapType.getValueType());
+      }
+    }
+    return null;
+  }
+
+  public String getRecursive(Map<String, Object> schema) {
+    String schemaStr = schema.toString();
+    for (FieldElement p : path) {
+      for (int i = 0; i < p.getSchemaTypes().size(); i++) {
+        if (p.getSchemaTypes().get(i).equals(schemaStr)) {
+          return p.getType().get(i);
+        }
+      }
+    }
+    return null;
+  }
+
+  public FieldPath popLast() {
+    FieldPath fpath = new FieldPath();
+    fpath.setKeySchema(isKeySchema);
+    fpath.setPath(new ArrayList<>(path));
+    fpath.getPath().remove(fpath.getPath().size() - 1);
+    return fpath;
+  }
+
+  public FieldPath clonePlus(FieldElement element) {
+    FieldPath fpath = new FieldPath();
+    fpath.setKeySchema(isKeySchema);
+    fpath.setPath(new ArrayList<>(path));
+    fpath.getPath().add(element);
+    return fpath;
+  }
+
+  // TODO: Why is typeSchema an Object?
+  public FieldPath expandType(String type, Object typeSchema) {
+    FieldPath fpath = new FieldPath();
+    fpath.setKeySchema(isKeySchema);
+    fpath.setPath(path.stream().map(FieldElement::clone).collect(Collectors.toList()));
+
+    if (!fpath.getPath().isEmpty()) {
+      FieldElement lastElement = fpath.getPath().get(fpath.getPath().size() - 1);
+      lastElement.getType().add(type);
+      lastElement.getSchemaTypes().add(typeSchema.toString());
+    } else {
+      fpath
+          .getPath()
+          .add(
+              new FieldElement(
+                  new ArrayList<>(Collections.singletonList(type)),
+                  new ArrayList<>(Collections.singletonList(typeSchema.toString())),
+                  null,
+                  null));
+    }
+    return fpath;
+  }
+
+  public boolean hasFieldName() {
+    return path.stream().anyMatch(f -> f.getName() != null);
+  }
+
+  public boolean ensureFieldName() {
+    if (!hasFieldName()) {
+      if (path.isEmpty()) {
+        path.add(new FieldElement(new ArrayList<>(), new ArrayList<>(), null, null));
+      }
+      path.get(path.size() - 1).setName(EMPTY_FIELD_NAME);
+    }
+    return true;
+  }
+
+  public String asString() {
+    boolean v2Format = needsV2Path();
+    List<String> prefix = new ArrayList<>();
+
+    if (v2Format) {
+      prefix.add("[version=2.0]");
+      if (isKeySchema) {
+        prefix.add("[key=True]");
+      }
+    }
+
+    if (!path.isEmpty()) {
+      return String.join(".", prefix)
+          + "."
+          + path.stream().map(f -> f.asString(v2Format)).collect(Collectors.joining("."));
+    } else {
+      return String.join(".", prefix);
+    }
+  }
+
+  public String dump() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("FieldPath: ");
+    sb.append(this.asString());
+    for (FieldElement f : path) {
+      sb.append(f.getName());
+      sb.append(" ");
+      sb.append(f.getSchemaTypes().toString());
+    }
+    return sb.toString();
+  }
+}
diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/utils/Constants.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/utils/Constants.java
new file mode 100644
index 0000000000000..b41d2d88c9dc0
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/utils/Constants.java
@@ -0,0 +1,12 @@
+package io.datahubproject.schematron.utils;
+
+/** Constants used throughout the schema conversion process. */
+public final class Constants {
+  private Constants() {}
+
+  public static final String ADD_TAG_OPERATION = "ADD_TAG";
+  public static final String ADD_TERM_OPERATION = "ADD_TERM";
+
+  public static final String TAG_URN_PREFIX = "urn:li:tag:";
+  public static final String TERM_URN_PREFIX = "urn:li:glossaryTerm:";
+}
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverterTest.java b/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverterTest.java
new file mode 100644
index 0000000000000..d6522c2d84670
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverterTest.java
@@ -0,0 +1,942 @@
+package io.datahubproject.schematron.converters.avro;
+
+import static org.testng.Assert.*;
+
+import com.linkedin.common.urn.DataPlatformUrn;
+import com.linkedin.data.template.StringArray;
+import com.linkedin.schema.*;
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import org.apache.avro.Schema;
+import org.testng.annotations.*;
+
+@Test(groups = "unit")
+class AvroSchemaConverterTest {
+
+  private AvroSchemaConverter avroSchemaConverter = AvroSchemaConverter.builder().build();
+  private DataPlatformUrn dataPlatformUrn =
+      DataPlatformUrn.createFromString("urn:li:dataPlatform:foo");
+
+  AvroSchemaConverterTest() throws URISyntaxException {}
+
+  @Test(groups = "basic")
+  void testPrimitiveTypes() throws IOException {
+    SchemaMetadata schema =
+        avroSchemaConverter.toDataHubSchema(
+            readAvroSchema("primitive_types.avsc"), false, false, dataPlatformUrn, null);
+
+    schema.getFields().forEach(System.out::println);
+
+    assertEquals(schema.getFields().size(), 14);
+
+    assertSchemaField(
+        schema.getFields().get(0),
+        "[version=2.0].[type=PrimitiveType].[type=int].intField",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(1),
+        "[version=2.0].[type=PrimitiveType].[type=union].intFieldV2",
+        "union",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new UnionType()
+                        .setNestedTypes(new StringArray(Collections.singletonList("union"))))));
+    assertSchemaField(
+        schema.getFields().get(2),
+        "[version=2.0].[type=PrimitiveType].[type=union].[type=int].intFieldV2",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(3),
+        "[version=2.0].[type=PrimitiveType].[type=null].nullField",
+        "null",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NullType())));
+    assertSchemaField(
+        schema.getFields().get(4),
+        "[version=2.0].[type=PrimitiveType].[type=union].nullFieldV2",
+        "union",
+        true,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new UnionType()
+                        .setNestedTypes(new StringArray(Collections.singletonList("union"))))));
+    assertSchemaField(
+        schema.getFields().get(5),
+        "[version=2.0].[type=PrimitiveType].[type=long].longField",
+        "long",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(6),
+        "[version=2.0].[type=PrimitiveType].[type=float].floatField",
+        "float",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(7),
+        "[version=2.0].[type=PrimitiveType].[type=double].doubleField",
+        "double",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(8),
+        "[version=2.0].[type=PrimitiveType].[type=string].stringField",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(9),
+        "[version=2.0].[type=PrimitiveType].[type=boolean].booleanField",
+        "boolean",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new BooleanType())));
+    assertSchemaField(
+        schema.getFields().get(10),
+        "[version=2.0].[type=PrimitiveType].[type=int].nullableIntField",
+        "int",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(11),
+        "[version=2.0].[type=PrimitiveType].[type=long].nullableLongField",
+        "long",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(12),
+        "[version=2.0].[type=PrimitiveType].[type=string].nullableStringField",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(13),
+        "[version=2.0].[type=PrimitiveType].[type=enum].status",
+        "Enum",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new EnumType())));
+  }
+
+  @Test(groups = "basic")
+  void testComplexMaps() throws IOException {
+    SchemaMetadata schema =
+        avroSchemaConverter.toDataHubSchema(
+            readAvroSchema("complex_maps.avsc"), false, false, dataPlatformUrn, null);
+
+    schema.getFields().forEach(System.out::println);
+
+    assertEquals(schema.getFields().size(), 15);
+
+    assertSchemaField(
+        schema.getFields().get(0),
+        "[version=2.0].[type=MapType].[type=map].mapOfString",
+        "map<string,string>",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("string"))));
+    assertSchemaField(
+        schema.getFields().get(1),
+        "[version=2.0].[type=MapType].[type=map].[type=ComplexType].mapOfComplexType",
+        "ComplexType",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("ComplexType"))));
+    assertSchemaField(
+        schema.getFields().get(2),
+        "[version=2.0].[type=MapType].[type=map].[type=ComplexType].mapOfComplexType.[type=string].field1",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(3),
+        "[version=2.0].[type=MapType].[type=map].[type=ComplexType].mapOfComplexType.[type=int].field2",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(4),
+        "[version=2.0].[type=MapType].[type=map].[type=union].mapOfNullableString",
+        "union",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("union"))));
+    assertSchemaField(
+        schema.getFields().get(5),
+        "[version=2.0].[type=MapType].[type=map].[type=union].[type=string].mapOfNullableString",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(6),
+        "[version=2.0].[type=MapType].[type=map].[type=union].mapOfNullableComplexType",
+        "union",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("union"))));
+    assertSchemaField(
+        schema.getFields().get(7),
+        "[version=2.0].[type=MapType].[type=map].[type=union].[type=ComplexTypeNullable].mapOfNullableComplexType",
+        "ComplexTypeNullable",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
+    assertSchemaField(
+        schema.getFields().get(8),
+        "[version=2.0].[type=MapType].[type=map].[type=union].[type=ComplexTypeNullable].mapOfNullableComplexType.[type=string].field1",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(9),
+        "[version=2.0].[type=MapType].[type=map].[type=union].[type=ComplexTypeNullable].mapOfNullableComplexType.[type=int].field2",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(10),
+        "[version=2.0].[type=MapType].[type=map].[type=array].mapOfArray",
+        "array(string)",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new ArrayType().setNestedType(new StringArray("string")))));
+    assertSchemaField(
+        schema.getFields().get(11),
+        "[version=2.0].[type=MapType].[type=map].[type=map].mapOfMap",
+        "map<string,int>",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("int"))));
+    assertSchemaField(
+        schema.getFields().get(12),
+        "[version=2.0].[type=MapType].[type=map].[type=union].mapOfUnion",
+        "union",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("union"))));
+    assertSchemaField(
+        schema.getFields().get(13),
+        "[version=2.0].[type=MapType].[type=map].[type=union].[type=string].mapOfUnion",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(14),
+        "[version=2.0].[type=MapType].[type=map].[type=union].[type=int].mapOfUnion",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+  }
+
+  @Test(groups = "basic")
+  void testComplexArrays() throws IOException {
+    SchemaMetadata schema =
+        avroSchemaConverter.toDataHubSchema(
+            readAvroSchema("complex_arrays.avsc"), false, false, dataPlatformUrn, null);
+
+    schema.getFields().forEach(System.out::println);
+
+    assertEquals(schema.getFields().size(), 16);
+
+    assertSchemaField(
+        schema.getFields().get(0),
+        "[version=2.0].[type=ArrayType].[type=array].arrayOfString",
+        "array(string)",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new ArrayType().setNestedType(new StringArray("string")))));
+    assertSchemaField(
+        schema.getFields().get(1),
+        "[version=2.0].[type=ArrayType].[type=array].[type=map].arrayOfMap",
+        "map<string,string>",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("string"))));
+    assertSchemaField(
+        schema.getFields().get(2),
+        "[version=2.0].[type=ArrayType].[type=array].[type=ComplexType].arrayOfRecord",
+        "ComplexType",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new ArrayType().setNestedType(new StringArray("ComplexType")))));
+    assertSchemaField(
+        schema.getFields().get(3),
+        "[version=2.0].[type=ArrayType].[type=array].[type=ComplexType].arrayOfRecord.[type=string].field1",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(4),
+        "[version=2.0].[type=ArrayType].[type=array].[type=ComplexType].arrayOfRecord.[type=int].field2",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(5),
+        "[version=2.0].[type=ArrayType].[type=array].[type=array].arrayOfArray",
+        "array(string)",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new ArrayType().setNestedType(new StringArray("string")))));
+    assertSchemaField(
+        schema.getFields().get(6),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].arrayOfUnion",
+        "union",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new ArrayType().setNestedType(new StringArray("union")))));
+    assertSchemaField(
+        schema.getFields().get(7),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].[type=string].arrayOfUnion",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(8),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].[type=int].arrayOfUnion",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(9),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].[type=boolean].arrayOfUnion",
+        "boolean",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new BooleanType())));
+    assertSchemaField(
+        schema.getFields().get(10),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].arrayOfNullableString",
+        "union",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new ArrayType().setNestedType(new StringArray("union")))));
+    assertSchemaField(
+        schema.getFields().get(11),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].[type=string].arrayOfNullableString",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(12),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].arrayOfNullableRecord",
+        "union",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new ArrayType().setNestedType(new StringArray("union")))));
+    assertSchemaField(
+        schema.getFields().get(13),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].[type=ComplexTypeNullable].arrayOfNullableRecord",
+        "ComplexTypeNullable",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
+    assertSchemaField(
+        schema.getFields().get(14),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].[type=ComplexTypeNullable].arrayOfNullableRecord.[type=string].field1",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(15),
+        "[version=2.0].[type=ArrayType].[type=array].[type=union].[type=ComplexTypeNullable].arrayOfNullableRecord.[type=int].field2",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+  }
+
+  @Test(groups = "basic")
+  void testComplexStructs() throws IOException {
+    SchemaMetadata schema =
+        avroSchemaConverter.toDataHubSchema(
+            readAvroSchema("complex_structs.avsc"), false, false, dataPlatformUrn, null);
+
+    schema.getFields().forEach(System.out::println);
+
+    assertEquals(schema.getFields().size(), 13);
+
+    assertSchemaField(
+        schema.getFields().get(0),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField",
+        "ComplexStruct",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
+    assertSchemaField(
+        schema.getFields().get(1),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=string].fieldString",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(2),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=int].fieldInt",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(3),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=boolean].fieldBoolean",
+        "boolean",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new BooleanType())));
+    assertSchemaField(
+        schema.getFields().get(4),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=map].fieldMap",
+        "map<string,string>",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("string"))));
+    assertSchemaField(
+        schema.getFields().get(5),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=NestedRecord].fieldRecord",
+        "NestedRecord",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
+    assertSchemaField(
+        schema.getFields().get(6),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=NestedRecord].fieldRecord.[type=string].nestedField1",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(7),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=NestedRecord].fieldRecord.[type=int].nestedField2",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(8),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=array].fieldArray",
+        "array(string)",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new ArrayType().setNestedType(new StringArray("string")))));
+    assertSchemaField(
+        schema.getFields().get(9),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=union].fieldUnion",
+        "union",
+        true,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new UnionType().setNestedTypes(new StringArray("union")))));
+    assertSchemaField(
+        schema.getFields().get(10),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=union].[type=string].fieldUnion",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(11),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=union].[type=int].fieldUnion",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(12),
+        "[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=map].fieldNullableMap",
+        "map<string,string>",
+        true,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("string"))));
+  }
+
+  @Test(groups = "basic")
+  void testComplexUnions() throws IOException {
+    SchemaMetadata schema =
+        avroSchemaConverter.toDataHubSchema(
+            readAvroSchema("complex_unions.avsc"), false, false, dataPlatformUrn, null);
+
+    schema.getFields().forEach(System.out::println);
+
+    assertEquals(schema.getFields().size(), 14);
+
+    assertSchemaField(
+        schema.getFields().get(0),
+        "[version=2.0].[type=UnionType].[type=union].fieldUnionNullablePrimitives",
+        "union",
+        true,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new UnionType().setNestedTypes(new StringArray("union")))));
+    assertSchemaField(
+        schema.getFields().get(1),
+        "[version=2.0].[type=UnionType].[type=union].[type=string].fieldUnionNullablePrimitives",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(2),
+        "[version=2.0].[type=UnionType].[type=union].[type=int].fieldUnionNullablePrimitives",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(3),
+        "[version=2.0].[type=UnionType].[type=union].[type=boolean].fieldUnionNullablePrimitives",
+        "boolean",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new BooleanType())));
+    assertSchemaField(
+        schema.getFields().get(4),
+        "[version=2.0].[type=UnionType].[type=union].fieldUnionComplexTypes",
+        "union",
+        true,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new UnionType().setNestedTypes(new StringArray("union")))));
+    assertSchemaField(
+        schema.getFields().get(5),
+        "[version=2.0].[type=UnionType].[type=union].[type=NestedRecord].fieldUnionComplexTypes",
+        "NestedRecord",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
+    assertSchemaField(
+        schema.getFields().get(6),
+        "[version=2.0].[type=UnionType].[type=union].[type=NestedRecord].fieldUnionComplexTypes.[type=string].nestedField1",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(7),
+        "[version=2.0].[type=UnionType].[type=union].[type=NestedRecord].fieldUnionComplexTypes.[type=int].nestedField2",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(8),
+        "[version=2.0].[type=UnionType].[type=union].[type=map].fieldUnionComplexTypes",
+        "map<string,string>",
+        false,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new MapType().setKeyType("string").setValueType("string"))));
+    assertSchemaField(
+        schema.getFields().get(9),
+        "[version=2.0].[type=UnionType].[type=union].fieldUnionPrimitiveAndComplex",
+        "union",
+        true,
+        false,
+        new SchemaFieldDataType()
+            .setType(
+                SchemaFieldDataType.Type.create(
+                    new UnionType().setNestedTypes(new StringArray("union")))));
+    assertSchemaField(
+        schema.getFields().get(10),
+        "[version=2.0].[type=UnionType].[type=union].[type=string].fieldUnionPrimitiveAndComplex",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(11),
+        "[version=2.0].[type=UnionType].[type=union].[type=ComplexTypeRecord].fieldUnionPrimitiveAndComplex",
+        "ComplexTypeRecord",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
+    assertSchemaField(
+        schema.getFields().get(12),
+        "[version=2.0].[type=UnionType].[type=union].[type=ComplexTypeRecord].fieldUnionPrimitiveAndComplex.[type=string].complexField1",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(13),
+        "[version=2.0].[type=UnionType].[type=union].[type=ComplexTypeRecord].fieldUnionPrimitiveAndComplex.[type=int].complexField2",
+        "int",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+  }
+
+  @Test(groups = "basic")
+  void testLogicalTypes() throws IOException {
+    SchemaMetadata schema =
+        avroSchemaConverter.toDataHubSchema(
+            readAvroSchema("logical_types.avsc"), false, false, dataPlatformUrn, null);
+
+    schema.getFields().forEach(System.out::println);
+
+    assertEquals(schema.getFields().size(), 9);
+
+    assertSchemaField(
+        schema.getFields().get(0),
+        "[version=2.0].[type=LogicalTypes].[type=bytes].decimalField",
+        "bytes(decimal)",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())),
+        "{\"scale\":2,\"logicalType\":\"decimal\",\"precision\":9}");
+    assertSchemaField(
+        schema.getFields().get(1),
+        "[version=2.0].[type=LogicalTypes].[type=bytes].decimalFieldWithoutScale",
+        "bytes(decimal)",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())),
+        "{\"logicalType\":\"decimal\",\"precision\":9}");
+    assertSchemaField(
+        schema.getFields().get(2),
+        "[version=2.0].[type=LogicalTypes].[type=bytes].decimalFieldWithoutPrecisionAndScale",
+        "bytes",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new BytesType())),
+        "{\"logicalType\":\"decimal\"}");
+    assertSchemaField(
+        schema.getFields().get(3),
+        "[version=2.0].[type=LogicalTypes].[type=long].timestampMillisField",
+        "long(timestamp-millis)",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
+        "{\"logicalType\":\"timestamp-millis\"}");
+    assertSchemaField(
+        schema.getFields().get(4),
+        "[version=2.0].[type=LogicalTypes].[type=long].timestampMicrosField",
+        "long(timestamp-micros)",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
+        "{\"logicalType\":\"timestamp-micros\"}");
+    assertSchemaField(
+        schema.getFields().get(5),
+        "[version=2.0].[type=LogicalTypes].[type=int].dateField",
+        "int(date)",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new DateType())),
+        "{\"logicalType\":\"date\"}");
+    assertSchemaField(
+        schema.getFields().get(6),
+        "[version=2.0].[type=LogicalTypes].[type=int].timeMillisField",
+        "int(time-millis)",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
+        "{\"logicalType\":\"time-millis\"}");
+    assertSchemaField(
+        schema.getFields().get(7),
+        "[version=2.0].[type=LogicalTypes].[type=long].timeMicrosField",
+        "long(time-micros)",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
+        "{\"logicalType\":\"time-micros\"}");
+    assertSchemaField(
+        schema.getFields().get(8),
+        "[version=2.0].[type=LogicalTypes].[type=string].uuidField",
+        "string(uuid)",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())),
+        "{\"logicalType\":\"uuid\"}");
+  }
+
+  @Test(groups = "basic")
+  void testUsersRecord() throws IOException {
+    // this is a test case got during the Hudi integration
+    SchemaMetadata schema =
+        avroSchemaConverter.toDataHubSchema(
+            readAvroSchema("users_record.avsc"), false, false, dataPlatformUrn, null);
+
+    schema.getFields().forEach(System.out::println);
+
+    assertEquals(schema.getFields().size(), 20);
+
+    assertSchemaField(
+        schema.getFields().get(0),
+        "[version=2.0].[type=users_record].[type=string]._hoodie_commit_time",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(1),
+        "[version=2.0].[type=users_record].[type=string]._hoodie_commit_seqno",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(2),
+        "[version=2.0].[type=users_record].[type=string]._hoodie_record_key",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(3),
+        "[version=2.0].[type=users_record].[type=string]._hoodie_partition_path",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(4),
+        "[version=2.0].[type=users_record].[type=string]._hoodie_file_name",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(5),
+        "[version=2.0].[type=users_record].[type=string].user_id",
+        "string",
+        false,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(6),
+        "[version=2.0].[type=users_record].[type=string].name",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(7),
+        "[version=2.0].[type=users_record].[type=address].address",
+        "address",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
+    assertSchemaField(
+        schema.getFields().get(8),
+        "[version=2.0].[type=users_record].[type=address].address.[type=string].street",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(9),
+        "[version=2.0].[type=users_record].[type=address].address.[type=string].city",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(10),
+        "[version=2.0].[type=users_record].[type=address].address.[type=string].country",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(11),
+        "[version=2.0].[type=users_record].[type=address].address.[type=string].postal_code",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(12),
+        "[version=2.0].[type=users_record].[type=address].address.[type=long].created_at",
+        "long(timestamp-micros)",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
+        "{\"logicalType\":\"timestamp-micros\"}");
+    assertSchemaField(
+        schema.getFields().get(13),
+        "[version=2.0].[type=users_record].[type=contact].contact",
+        "contact",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
+    assertSchemaField(
+        schema.getFields().get(14),
+        "[version=2.0].[type=users_record].[type=contact].contact.[type=string].email",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(15),
+        "[version=2.0].[type=users_record].[type=contact].contact.[type=string].phone",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+    assertSchemaField(
+        schema.getFields().get(16),
+        "[version=2.0].[type=users_record].[type=long].created_at",
+        "long(timestamp-micros)",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
+        "{\"logicalType\":\"timestamp-micros\"}");
+    assertSchemaField(
+        schema.getFields().get(17),
+        "[version=2.0].[type=users_record].[type=long].updated_at",
+        "long(timestamp-micros)",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
+        "{\"logicalType\":\"timestamp-micros\"}");
+    assertSchemaField(
+        schema.getFields().get(18),
+        "[version=2.0].[type=users_record].[type=map].[type=int].props",
+        "int",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
+    assertSchemaField(
+        schema.getFields().get(19),
+        "[version=2.0].[type=users_record].[type=string].country",
+        "string",
+        true,
+        false,
+        new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
+  }
+
+  private void assertSchemaField(
+      SchemaField field,
+      String expectedPath,
+      String expectedNativeType,
+      boolean expectedNullable,
+      boolean expectedIsPartOfKey,
+      SchemaFieldDataType expectedType) {
+    assertSchemaField(
+        field,
+        expectedPath,
+        expectedNativeType,
+        expectedNullable,
+        expectedIsPartOfKey,
+        expectedType,
+        null);
+  }
+
+  private void assertSchemaField(
+      SchemaField field,
+      String expectedPath,
+      String expectedNativeType,
+      boolean expectedNullable,
+      boolean expectedIsPartOfKey,
+      SchemaFieldDataType expectedType,
+      String expectedJsonProps) {
+    assertEquals(field.getFieldPath(), expectedPath);
+    assertEquals(field.getNativeDataType(), expectedNativeType);
+    assertEquals(field.isNullable(), expectedNullable);
+    assertEquals(field.isIsPartOfKey(), expectedIsPartOfKey);
+    assertEquals(field.getType(), expectedType);
+    if (expectedJsonProps != null) {
+      assertEquals(field.getJsonProps(), expectedJsonProps);
+    }
+  }
+
+  private Schema readAvroSchema(String schemaFileName) throws IOException {
+    String schemaPath = getClass().getClassLoader().getResource(schemaFileName).getPath();
+    File schemaFile = new File(schemaPath);
+    return new Schema.Parser().parse(schemaFile);
+  }
+}
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/models/FieldPathTest.java b/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/models/FieldPathTest.java
new file mode 100644
index 0000000000000..d823a2c8ed51b
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/models/FieldPathTest.java
@@ -0,0 +1,246 @@
+package io.datahubproject.schematron.models;
+
+import static org.testng.Assert.*;
+
+import com.linkedin.schema.ArrayType;
+import com.linkedin.schema.MapType;
+import com.linkedin.schema.UnionType;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.testng.annotations.*;
+
+@Test(groups = "unit")
+public class FieldPathTest {
+
+  @Test(groups = "basic")
+  public void testEmptyFieldPath() {
+    FieldPath path = new FieldPath();
+    assertEquals(path.asString(), "[version=2.0]");
+  }
+
+  @Test(groups = "basic")
+  public void testKeySchemaPath() {
+    FieldPath path = new FieldPath();
+    path.setKeySchema(true);
+    assertEquals(path.asString(), "[version=2.0].[key=True]");
+  }
+
+  @Test(groups = "basic")
+  public void testSimplePath() {
+    FieldPath path = new FieldPath();
+    FieldElement element =
+        new FieldElement(
+            Collections.singletonList("string"), Collections.singletonList("schema"), "name", null);
+    path.setPath(Collections.singletonList(element));
+    assertEquals(path.asString(), "[version=2.0].[type=string].name");
+  }
+
+  @Test(groups = "nested")
+  public void testNestedPath() {
+    FieldPath path = new FieldPath();
+    FieldElement record =
+        new FieldElement(
+            Collections.singletonList("record"),
+            Collections.singletonList("record-schema"),
+            "user",
+            null);
+    FieldElement field =
+        new FieldElement(
+            Collections.singletonList("string"),
+            Collections.singletonList("string-schema"),
+            "name",
+            null);
+    path.setPath(Arrays.asList(record, field));
+    assertEquals(path.asString(), "[version=2.0].[type=record].user.[type=string].name");
+  }
+
+  @Test(groups = "complex")
+  public void testUnionPath() {
+    FieldPath path = new FieldPath();
+
+    // Add union type
+    FieldElement union =
+        new FieldElement(
+            Collections.singletonList("union"),
+            Collections.singletonList("union-schema"),
+            "document",
+            null);
+
+    // Add specific union member (record type)
+    FieldElement passport =
+        new FieldElement(
+            Collections.singletonList("Passport"),
+            Collections.singletonList("passport-schema"),
+            "document",
+            new DataHubType(UnionType.class, "Passport"));
+
+    // Add field within the record
+    FieldElement number =
+        new FieldElement(
+            Collections.singletonList("string"),
+            Collections.singletonList("string-schema"),
+            "number",
+            null);
+
+    path.setPath(Arrays.asList(union, passport, number));
+    assertEquals(
+        path.asString(),
+        "[version=2.0].[type=union].document.[type=Passport].document.[type=string].number");
+  }
+
+  @Test(groups = "operations")
+  public void testClonePlus() {
+    FieldPath original = new FieldPath();
+    FieldElement element1 =
+        new FieldElement(
+            Collections.singletonList("record"),
+            Collections.singletonList("schema1"),
+            "user",
+            null);
+    original.setPath(Collections.singletonList(element1));
+
+    FieldElement element2 =
+        new FieldElement(
+            Collections.singletonList("string"),
+            Collections.singletonList("schema2"),
+            "name",
+            null);
+
+    FieldPath newPath = original.clonePlus(element2);
+
+    // Verify original path remains unchanged
+    assertEquals(original.asString(), "[version=2.0].[type=record].user");
+
+    // Verify new path has both elements
+    assertEquals(newPath.asString(), "[version=2.0].[type=record].user.[type=string].name");
+  }
+
+  @Test(groups = "operations")
+  public void testExpandType() {
+    FieldPath path = new FieldPath();
+    FieldElement element = new FieldElement(new ArrayList<>(), new ArrayList<>(), "field", null);
+    path.setPath(Collections.singletonList(element));
+
+    FieldPath expanded = path.expandType("string", "schema");
+
+    assertEquals(expanded.asString(), "[version=2.0].[type=string].field");
+    assertEquals(expanded.getPath().get(0).getType().size(), 1);
+    assertEquals(expanded.getPath().get(0).getType().get(0), "string");
+    assertEquals(expanded.getPath().get(0).getSchemaTypes().get(0), "schema");
+  }
+
+  @Test(groups = "operations")
+  public void testHasFieldName() {
+    FieldPath path = new FieldPath();
+    assertFalse(path.hasFieldName());
+
+    FieldElement element =
+        new FieldElement(
+            Collections.singletonList("string"), Collections.singletonList("schema"), "name", null);
+    path.setPath(Collections.singletonList(element));
+    assertTrue(path.hasFieldName());
+  }
+
+  @Test(groups = "operations")
+  public void testEnsureFieldName() {
+    FieldPath path = new FieldPath();
+    assertFalse(path.hasFieldName());
+
+    path.ensureFieldName();
+    assertTrue(path.hasFieldName());
+    assertEquals(path.getPath().get(0).getName(), FieldPath.EMPTY_FIELD_NAME);
+  }
+
+  @Test(groups = "complex")
+  public void testArrayPath() {
+    FieldPath path = new FieldPath();
+    FieldElement array =
+        new FieldElement(
+            Collections.singletonList("array"),
+            Collections.singletonList("array-schema"),
+            "items",
+            new DataHubType(ArrayType.class, "string"));
+
+    path.setPath(Collections.singletonList(array));
+    assertEquals(path.asString(), "[version=2.0].[type=array].items");
+  }
+
+  @Test(groups = "complex")
+  public void testMapPath() {
+    FieldPath path = new FieldPath();
+    FieldElement map =
+        new FieldElement(
+            Collections.singletonList("map"),
+            Collections.singletonList("map-schema"),
+            "properties",
+            new DataHubType(MapType.class, "string"));
+
+    path.setPath(Collections.singletonList(map));
+    assertEquals(path.asString(), "[version=2.0].[type=map].properties");
+  }
+
+  @Test(groups = "complex")
+  public void testMultipleTypesInPath() {
+    FieldPath path = new FieldPath();
+    FieldElement element =
+        new FieldElement(
+            Arrays.asList("union", "string"),
+            Arrays.asList("union-schema", "string-schema"),
+            "field",
+            null);
+    path.setPath(Collections.singletonList(element));
+    assertEquals(path.asString(), "[version=2.0].[type=union].[type=string].field");
+  }
+
+  @Test(groups = "complex")
+  public void testParentTypeHandling() {
+    FieldPath path = new FieldPath();
+    DataHubType parentType = new DataHubType(ArrayType.class, "string");
+    FieldElement element =
+        new FieldElement(
+            Collections.singletonList("array"),
+            Collections.singletonList("array-schema"),
+            "items",
+            parentType);
+    path.setPath(Collections.singletonList(element));
+
+    assertNotNull(path.getPath().get(0).getParentType());
+    assertEquals(path.getPath().get(0).getParentType().getType(), ArrayType.class);
+    assertEquals(path.getPath().get(0).getParentType().getNestedType(), "string");
+  }
+
+  @Test(groups = "edge-cases")
+  public void testNoParentPath() {
+    FieldPath path = new FieldPath();
+    assertEquals(path.asString(), "[version=2.0]");
+  }
+
+  @Test(groups = "edge-cases")
+  public void testEmptyElementList() {
+    FieldPath path = new FieldPath();
+    path.setPath(new ArrayList<>());
+    assertEquals(path.asString(), "[version=2.0]");
+  }
+
+  @DataProvider(name = "invalidPaths")
+  public Object[][] getInvalidPaths() {
+    return new Object[][] {
+      {null, "Expected IllegalArgumentException for null element"},
+      {
+        Arrays.asList((FieldElement) null),
+        "Expected IllegalArgumentException for null element in list"
+      }
+    };
+  }
+
+  @Test(
+      groups = "edge-cases",
+      dataProvider = "invalidPaths",
+      expectedExceptions = IllegalArgumentException.class)
+  public void testInvalidPaths(List<FieldElement> elements, String message) {
+    FieldPath path = new FieldPath();
+    path.setPath(elements);
+  }
+}
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_arrays.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_arrays.avsc
new file mode 100644
index 0000000000000..8e8bcdaa0a7dc
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_arrays.avsc
@@ -0,0 +1,87 @@
+{
+  "type": "record",
+  "name": "ArrayType",
+  "fields": [
+    {
+      "name": "arrayOfString",
+      "type": {
+        "type": "array",
+        "items": "string"
+      }
+    },
+    {
+      "name": "arrayOfMap",
+      "type": {
+        "type": "array",
+        "items": {
+          "type": "map",
+          "values": "string"
+        }
+      }
+    },
+    {
+      "name": "arrayOfRecord",
+      "type": {
+        "type": "array",
+        "items": {
+          "type": "record",
+          "name": "ComplexType",
+          "fields": [
+            {
+              "name": "field1",
+              "type": "string"
+            },
+            {
+              "name": "field2",
+              "type": "int"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "name": "arrayOfArray",
+      "type": {
+        "type": "array",
+        "items": {
+          "type": "array",
+          "items": "string"
+        }
+      }
+    },
+    {
+      "name": "arrayOfUnion",
+      "type": {
+        "type": "array",
+        "items": ["string", "int", "boolean"]
+      }
+    },
+    {
+      "name": "arrayOfNullableString",
+      "type": {
+        "type": "array",
+        "items": ["null", "string"]
+      }
+    },
+    {
+      "name": "arrayOfNullableRecord",
+      "type": {
+        "type": "array",
+        "items": ["null", {
+          "type": "record",
+          "name": "ComplexTypeNullable",
+          "fields": [
+            {
+              "name": "field1",
+              "type": "string"
+            },
+            {
+              "name": "field2",
+              "type": "int"
+            }
+          ]
+        }]
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_maps.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_maps.avsc
new file mode 100644
index 0000000000000..baedae1b9dcc1
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_maps.avsc
@@ -0,0 +1,87 @@
+{
+  "type": "record",
+  "name": "MapType",
+  "fields": [
+    {
+      "name": "mapOfString",
+      "type": {
+        "type": "map",
+        "values": "string"
+      }
+    },
+    {
+      "name": "mapOfComplexType",
+      "type": {
+        "type": "map",
+        "values": {
+          "type": "record",
+          "name": "ComplexType",
+          "fields": [
+            {
+              "name": "field1",
+              "type": "string"
+            },
+            {
+              "name": "field2",
+              "type": "int"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "name": "mapOfNullableString",
+      "type": {
+        "type": "map",
+        "values": ["null", "string"]
+      }
+    },
+    {
+      "name": "mapOfNullableComplexType",
+      "type": {
+        "type": "map",
+        "values": ["null", {
+          "type": "record",
+          "name": "ComplexTypeNullable",
+          "fields": [
+            {
+              "name": "field1",
+              "type": "string"
+            },
+            {
+              "name": "field2",
+              "type": "int"
+            }
+          ]
+        }]
+      }
+    },
+    {
+      "name": "mapOfArray",
+      "type": {
+        "type": "map",
+        "values": {
+          "type": "array",
+          "items": "string"
+        }
+      }
+    },
+    {
+      "name": "mapOfMap",
+      "type": {
+        "type": "map",
+        "values": {
+          "type": "map",
+          "values": "int"
+        }
+      }
+    },
+    {
+      "name": "mapOfUnion",
+      "type": {
+        "type": "map",
+        "values": ["null", "string", "int"]
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_structs.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_structs.avsc
new file mode 100644
index 0000000000000..7f5824192d306
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_structs.avsc
@@ -0,0 +1,76 @@
+{
+  "type": "record",
+  "name": "StructType",
+  "fields": [
+    {
+      "name": "structField",
+      "type": {
+        "type": "record",
+        "name": "ComplexStruct",
+        "fields": [
+          {
+            "name": "fieldString",
+            "type": "string"
+          },
+          {
+            "name": "fieldInt",
+            "type": "int"
+          },
+          {
+            "name": "fieldBoolean",
+            "type": "boolean"
+          },
+          {
+            "name": "fieldMap",
+            "type": {
+              "type": "map",
+              "values": "string"
+            }
+          },
+          {
+            "name": "fieldRecord",
+            "type": {
+              "type": "record",
+              "name": "NestedRecord",
+              "fields": [
+                {
+                  "name": "nestedField1",
+                  "type": "string"
+                },
+                {
+                  "name": "nestedField2",
+                  "type": "int"
+                }
+              ]
+            }
+          },
+          {
+            "name": "fieldArray",
+            "type": {
+              "type": "array",
+              "items": "string"
+            }
+          },
+          {
+            "name": "fieldUnion",
+            "type": [
+              "null",
+              "string",
+              "int"
+            ]
+          },
+          {
+            "name": "fieldNullableMap",
+            "type": [
+              "null",
+              {
+                "type": "map",
+                "values": "string"
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_unions.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_unions.avsc
new file mode 100644
index 0000000000000..1a35f1cfa0e6d
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_unions.avsc
@@ -0,0 +1,60 @@
+{
+  "type": "record",
+  "name": "UnionType",
+  "fields": [
+    {
+      "name": "fieldUnionNullablePrimitives",
+      "type": [
+        "null",
+        "string",
+        "int",
+        "boolean"
+      ]
+    },
+    {
+      "name": "fieldUnionComplexTypes",
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "NestedRecord",
+          "fields": [
+            {
+              "name": "nestedField1",
+              "type": "string"
+            },
+            {
+              "name": "nestedField2",
+              "type": "int"
+            }
+          ]
+        },
+        {
+          "type": "map",
+          "values": "string"
+        }
+      ]
+    },
+    {
+      "name": "fieldUnionPrimitiveAndComplex",
+      "type": [
+        "null",
+        "string",
+        {
+          "type": "record",
+          "name": "ComplexTypeRecord",
+          "fields": [
+            {
+              "name": "complexField1",
+              "type": "string"
+            },
+            {
+              "name": "complexField2",
+              "type": "int"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/logical_types.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/logical_types.avsc
new file mode 100644
index 0000000000000..24919d8214965
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/logical_types.avsc
@@ -0,0 +1,72 @@
+{
+  "type": "record",
+  "name": "LogicalTypes",
+  "fields": [
+    {
+      "name": "decimalField",
+      "type": {
+        "type": "bytes",
+        "logicalType": "decimal",
+        "precision": 9,
+        "scale": 2
+      }
+    },
+    {
+      "name": "decimalFieldWithoutScale",
+      "type": {
+        "type": "bytes",
+        "logicalType": "decimal",
+        "precision": 9
+      }
+    },
+    {
+      "name": "decimalFieldWithoutPrecisionAndScale",
+      "type": {
+        "type": "bytes",
+        "logicalType": "decimal"
+      }
+    },
+    {
+      "name": "timestampMillisField",
+      "type": {
+        "type": "long",
+        "logicalType": "timestamp-millis"
+      }
+    },
+    {
+      "name": "timestampMicrosField",
+      "type": {
+        "type": "long",
+        "logicalType": "timestamp-micros"
+      }
+    },
+    {
+      "name": "dateField",
+      "type": {
+        "type": "int",
+        "logicalType": "date"
+      }
+    },
+    {
+      "name": "timeMillisField",
+      "type": {
+        "type": "int",
+        "logicalType": "time-millis"
+      }
+    },
+    {
+      "name": "timeMicrosField",
+      "type": {
+        "type": "long",
+        "logicalType": "time-micros"
+      }
+    },
+    {
+      "name": "uuidField",
+      "type": {
+        "type": "string",
+        "logicalType": "uuid"
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/primitive_types.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/primitive_types.avsc
new file mode 100644
index 0000000000000..c618299748fab
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/primitive_types.avsc
@@ -0,0 +1,62 @@
+{
+  "type": "record",
+  "name": "PrimitiveType",
+  "fields": [
+    {
+      "name": "intField",
+      "type": "int"
+    },
+    {
+      "name": "intFieldV2",
+      "type": ["int"]
+    },
+    {
+      "name": "nullField",
+      "type": "null"
+    },
+    {
+      "name": "nullFieldV2",
+      "type": ["null"]
+    },
+    {
+      "name": "longField",
+      "type": "long"
+    },
+    {
+      "name": "floatField",
+      "type": "float"
+    },
+    {
+      "name": "doubleField",
+      "type": "double"
+    },
+    {
+      "name": "stringField",
+      "type": "string"
+    },
+    {
+      "name": "booleanField",
+      "type": "boolean"
+    },
+    {
+      "name": "nullableIntField",
+      "type": ["null", "int"]
+    },
+    {
+      "name": "nullableLongField",
+      "type": ["null", "long"]
+    },
+    {
+      "name": "nullableStringField",
+      "type": ["null", "string"]
+    },
+    {
+      "name": "status",
+      "type": {
+        "type": "enum",
+        "name": "StatusEnum",
+        "symbols": ["ACTIVE", "INACTIVE", "PENDING"]
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/users_record.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/users_record.avsc
new file mode 100644
index 0000000000000..bd46ae715a481
--- /dev/null
+++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/users_record.avsc
@@ -0,0 +1,195 @@
+{
+  "type": "record",
+  "name": "users_record",
+  "namespace": "hoodie.users",
+  "fields": [
+    {
+      "name": "_hoodie_commit_time",
+      "type": [
+        "null",
+        "string"
+      ],
+      "doc": "",
+      "default": null
+    },
+    {
+      "name": "_hoodie_commit_seqno",
+      "type": [
+        "null",
+        "string"
+      ],
+      "doc": "",
+      "default": null
+    },
+    {
+      "name": "_hoodie_record_key",
+      "type": [
+        "null",
+        "string"
+      ],
+      "doc": "",
+      "default": null
+    },
+    {
+      "name": "_hoodie_partition_path",
+      "type": [
+        "null",
+        "string"
+      ],
+      "doc": "",
+      "default": null
+    },
+    {
+      "name": "_hoodie_file_name",
+      "type": [
+        "null",
+        "string"
+      ],
+      "doc": "",
+      "default": null
+    },
+    {
+      "name": "user_id",
+      "type": "string"
+    },
+    {
+      "name": "name",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "address",
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "address",
+          "namespace": "hoodie.users.users_record",
+          "fields": [
+            {
+              "name": "street",
+              "type": [
+                "null",
+                "string"
+              ],
+              "default": null
+            },
+            {
+              "name": "city",
+              "type": [
+                "null",
+                "string"
+              ],
+              "default": null
+            },
+            {
+              "name": "country",
+              "type": [
+                "null",
+                "string"
+              ],
+              "default": null
+            },
+            {
+              "name": "postal_code",
+              "type": [
+                "null",
+                "string"
+              ],
+              "default": null
+            },
+            {
+              "name": "created_at",
+              "type": [
+                "null",
+                {
+                  "type": "long",
+                  "logicalType": "timestamp-micros"
+                }
+              ],
+              "default": null
+            }
+          ]
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "contact",
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "contact",
+          "namespace": "hoodie.users.users_record",
+          "fields": [
+            {
+              "name": "email",
+              "type": [
+                "null",
+                "string"
+              ],
+              "default": null
+            },
+            {
+              "name": "phone",
+              "type": [
+                "null",
+                "string"
+              ],
+              "default": null
+            }
+          ]
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "created_at",
+      "type": [
+        "null",
+        {
+          "type": "long",
+          "logicalType": "timestamp-micros"
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "updated_at",
+      "type": [
+        "null",
+        {
+          "type": "long",
+          "logicalType": "timestamp-micros"
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "props",
+      "type": [
+        "null",
+        {
+          "type": "map",
+          "values": [
+            "null",
+            "int"
+          ]
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "country",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    }
+  ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/openlineage-converter/build.gradle b/metadata-integration/java/openlineage-converter/build.gradle
index 2e04881ab5ccd..301d1e6dffdd8 100644
--- a/metadata-integration/java/openlineage-converter/build.gradle
+++ b/metadata-integration/java/openlineage-converter/build.gradle
@@ -1,8 +1,8 @@
-apply plugin: 'java'
-apply plugin: 'com.github.johnrengelman.shadow'
+apply plugin: 'java-library'
+apply plugin: 'com.gradleup.shadow'
 apply plugin: 'signing'
 apply plugin: 'maven-publish'
-apply plugin: 'jacoco'
+apply from: '../../../gradle/coverage/java-coverage.gradle'
 apply from: '../versioning.gradle'
 
 repositories {
@@ -27,20 +27,16 @@ dependencies {
     testImplementation externalDependency.testng
 }
 
-jacocoTestReport {
-    dependsOn test // tests are required to run before generating the report
-}
-
 test {
     forkEvery = 1
     useJUnit()
-    finalizedBy jacocoTestReport
 }
 
 test {
     useJUnit()
-    finalizedBy jacocoTestReport
+    useTestNG()
 }
+
 shadowJar {
     zip64 = true
     archiveClassifier = ''
diff --git a/metadata-integration/java/spark-lineage-legacy/build.gradle b/metadata-integration/java/spark-lineage-legacy/build.gradle
index 8db8a09f8cc81..f0281a685368e 100644
--- a/metadata-integration/java/spark-lineage-legacy/build.gradle
+++ b/metadata-integration/java/spark-lineage-legacy/build.gradle
@@ -2,11 +2,10 @@ plugins {
     id("com.palantir.git-version") apply false
 }
 apply plugin: 'java'
-apply plugin: 'com.github.johnrengelman.shadow'
+apply plugin: 'com.gradleup.shadow'
 apply plugin: 'signing'
 apply plugin: 'io.codearte.nexus-staging'
 apply plugin: 'maven-publish'
-apply plugin: 'jacoco'
 apply from: '../versioning.gradle'
 
 jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
@@ -123,15 +122,6 @@ shadowJar {
   finalizedBy checkShadowJar
 }
 
-checkShadowJar {
-   dependsOn shadowJar
-}
-
-
-jacocoTestReport {
-    dependsOn test // tests are required to run before generating the report
-}
-
 test {
   forkEvery = 1
   useJUnit()
@@ -154,7 +144,6 @@ test {
   ]
   jvmArgs = sparkJava17CompatibleJvmArgs
 
-  finalizedBy jacocoTestReport
 }
 
 assemble {
diff --git a/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh b/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh
index 81d6a541d1c2a..d410842121648 100755
--- a/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh
+++ b/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh
@@ -40,7 +40,10 @@ jar -tvf $jarFile |\
       grep -v "rootdoc.txt" |\
       grep -v "VersionInfo.java" |\
       grep -v "mime.types" |\
-      grep -v "com/ibm/.*"
+      grep -v "com/ibm/.*" |\
+      grep -v "google/" |\
+      grep -v "org/apache/avro" |\
+      grep -v "org/apache"
 
 
 if [ $? -ne 0 ]; then
diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle
index 09a41d100199d..516a77d59d50b 100644
--- a/metadata-io/build.gradle
+++ b/metadata-io/build.gradle
@@ -4,6 +4,8 @@ plugins {
   id 'io.ebean' version "${ebeanVersion}" // Use the latest version from global build.gradle
 }
 
+apply from: '../gradle/coverage/java-coverage.gradle'
+
 configurations {
   enhance
 }
@@ -140,6 +142,8 @@ test {
   }
   testLogging.showStandardStreams = true
   testLogging.exceptionFormat = 'full'
+
+  environment 'STRICT_URN_VALIDATION_ENABLED', 'true'
 }
 
 ebean {
diff --git a/metadata-io/metadata-io-api/build.gradle b/metadata-io/metadata-io-api/build.gradle
index b8028fad07bb6..a49da206a69a8 100644
--- a/metadata-io/metadata-io-api/build.gradle
+++ b/metadata-io/metadata-io-api/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java-library'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(':entity-registry')
   implementation project(':metadata-service:services')
@@ -16,3 +18,7 @@ dependencies {
   testImplementation externalDependency.lombok
   testAnnotationProcessor externalDependency.lombok
 }
+
+test {
+  environment 'STRICT_URN_VALIDATION_ENABLED', 'true'
+}
\ No newline at end of file
diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityAspect.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityAspect.java
index cba770d841b94..2b67d5e92f833 100644
--- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityAspect.java
+++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityAspect.java
@@ -9,6 +9,7 @@
 import com.linkedin.metadata.aspect.SystemAspect;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.mxe.GenericAspect;
 import com.linkedin.mxe.SystemMetadata;
 import java.sql.Timestamp;
 import javax.annotation.Nonnull;
@@ -51,6 +52,26 @@ public class EntityAspect {
 
   private String createdFor;
 
+  @Override
+  public String toString() {
+    return "EntityAspect{"
+        + "urn='"
+        + urn
+        + '\''
+        + ", aspect='"
+        + aspect
+        + '\''
+        + ", version="
+        + version
+        + ", metadata='"
+        + metadata
+        + '\''
+        + ", systemMetadata='"
+        + systemMetadata
+        + '\''
+        + '}';
+  }
+
   /**
    * Provide a typed EntityAspect without breaking the existing public contract with generic types.
    */
@@ -65,7 +86,7 @@ public static class EntitySystemAspect implements SystemAspect {
     @Nullable private final RecordTemplate recordTemplate;
 
     @Nonnull private final EntitySpec entitySpec;
-    @Nonnull private final AspectSpec aspectSpec;
+    @Nullable private final AspectSpec aspectSpec;
 
     @Nonnull
     public String getUrnRaw() {
@@ -143,6 +164,11 @@ public EnvelopedAspect toEnvelopedAspects() {
       return envelopedAspect;
     }
 
+    @Override
+    public String toString() {
+      return entityAspect.toString();
+    }
+
     public static class EntitySystemAspectBuilder {
 
       private EntityAspect.EntitySystemAspect build() {
@@ -151,7 +177,7 @@ private EntityAspect.EntitySystemAspect build() {
 
       public EntityAspect.EntitySystemAspect build(
           @Nonnull EntitySpec entitySpec,
-          @Nonnull AspectSpec aspectSpec,
+          @Nullable AspectSpec aspectSpec,
           @Nonnull EntityAspect entityAspect) {
         this.entityAspect = entityAspect;
         this.urn = UrnUtils.getUrn(entityAspect.getUrn());
@@ -159,7 +185,11 @@ public EntityAspect.EntitySystemAspect build(
         if (entityAspect.getMetadata() != null) {
           this.recordTemplate =
               RecordUtils.toRecordTemplate(
-                  aspectSpec.getDataTemplateClass(), entityAspect.getMetadata());
+                  (Class<? extends RecordTemplate>)
+                      (aspectSpec == null
+                          ? GenericAspect.class
+                          : aspectSpec.getDataTemplateClass()),
+                  entityAspect.getMetadata());
         }
 
         return new EntitySystemAspect(entityAspect, urn, recordTemplate, entitySpec, aspectSpec);
diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java
index 1fba842631720..1af9fc1565a45 100644
--- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java
+++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java
@@ -1,8 +1,10 @@
 package com.linkedin.metadata.entity.ebean.batch;
 
 import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.aspect.RetrieverContext;
 import com.linkedin.metadata.aspect.SystemAspect;
 import com.linkedin.metadata.aspect.batch.AspectsBatch;
@@ -11,9 +13,12 @@
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.plugins.hooks.MutationHook;
 import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection;
+import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.util.Pair;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -27,12 +32,23 @@
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-@Getter
 @Builder(toBuilder = true)
 public class AspectsBatchImpl implements AspectsBatch {
 
   @Nonnull private final Collection<? extends BatchItem> items;
-  @Nonnull private final RetrieverContext retrieverContext;
+  @Nonnull private final Collection<? extends BatchItem> nonRepeatedItems;
+  @Getter @Nonnull private final RetrieverContext retrieverContext;
+
+  @Override
+  @Nonnull
+  public Collection<? extends BatchItem> getItems() {
+    return nonRepeatedItems;
+  }
+
+  @Override
+  public Collection<? extends BatchItem> getInitialItems() {
+    return items;
+  }
 
   /**
    * Convert patches to upserts, apply hooks at the aspect and batch level.
@@ -45,7 +61,8 @@ public class AspectsBatchImpl implements AspectsBatch {
    */
   @Override
   public Pair<Map<String, Set<String>>, List<ChangeMCP>> toUpsertBatchItems(
-      final Map<String, Map<String, SystemAspect>> latestAspects) {
+      Map<String, Map<String, SystemAspect>> latestAspects,
+      Map<String, Map<String, Long>> nextVersions) {
 
     // Process proposals to change items
     Stream<? extends BatchItem> mutatedProposalsStream =
@@ -54,6 +71,7 @@ public Pair<Map<String, Set<String>>, List<ChangeMCP>> toUpsertBatchItems(
                 .filter(item -> item instanceof ProposedItem)
                 .map(item -> (MCPItem) item)
                 .collect(Collectors.toList()));
+
     // Regular change items
     Stream<? extends BatchItem> changeMCPStream =
         items.stream().filter(item -> !(item instanceof ProposedItem));
@@ -81,10 +99,8 @@ public Pair<Map<String, Set<String>>, List<ChangeMCP>> toUpsertBatchItems(
                             currentValue, retrieverContext.getAspectRetriever());
                   }
 
-                  // Populate old aspect for write hooks
-                  upsertItem.setPreviousSystemAspect(latest);
-
-                  return upsertItem;
+                  return AspectsBatch.incrementBatchVersion(
+                      upsertItem, latestAspects, nextVersions);
                 })
             .collect(Collectors.toCollection(LinkedList::new));
 
@@ -94,6 +110,7 @@ public Pair<Map<String, Set<String>>, List<ChangeMCP>> toUpsertBatchItems(
     LinkedList<ChangeMCP> newItems =
         applyMCPSideEffects(upsertBatchItems).collect(Collectors.toCollection(LinkedList::new));
     upsertBatchItems.addAll(newItems);
+
     Map<String, Set<String>> newUrnAspectNames =
         getNewUrnAspectsMap(getUrnAspectsMap(), upsertBatchItems);
 
@@ -114,19 +131,7 @@ private Stream<? extends BatchItem> proposedItemsToChangeItemStream(List<MCPItem
                                     proposedItem.getChangeType(),
                                     proposedItem.getUrn(),
                                     proposedItem.getAspectName())))
-            .map(
-                mcpItem -> {
-                  if (ChangeType.PATCH.equals(mcpItem.getChangeType())) {
-                    return PatchItemImpl.PatchItemImplBuilder.build(
-                        mcpItem.getMetadataChangeProposal(),
-                        mcpItem.getAuditStamp(),
-                        retrieverContext.getAspectRetriever().getEntityRegistry());
-                  }
-                  return ChangeItemImpl.ChangeItemImplBuilder.build(
-                      mcpItem.getMetadataChangeProposal(),
-                      mcpItem.getAuditStamp(),
-                      retrieverContext.getAspectRetriever());
-                });
+            .map(mcpItem -> patchDiscriminator(mcpItem, retrieverContext.getAspectRetriever()));
     List<MCPItem> mutatedItems =
         applyProposalMutationHooks(proposedItems, retrieverContext).collect(Collectors.toList());
     Stream<? extends BatchItem> proposedItemsToChangeItems =
@@ -134,12 +139,7 @@ private Stream<? extends BatchItem> proposedItemsToChangeItemStream(List<MCPItem
             .filter(mcpItem -> mcpItem.getMetadataChangeProposal() != null)
             // Filter on proposed items again to avoid applying builder to Patch Item side effects
             .filter(mcpItem -> mcpItem instanceof ProposedItem)
-            .map(
-                mcpItem ->
-                    ChangeItemImpl.ChangeItemImplBuilder.build(
-                        mcpItem.getMetadataChangeProposal(),
-                        mcpItem.getAuditStamp(),
-                        retrieverContext.getAspectRetriever()));
+            .map(mcpItem -> patchDiscriminator(mcpItem, retrieverContext.getAspectRetriever()));
     Stream<? extends BatchItem> sideEffectItems =
         mutatedItems.stream().filter(mcpItem -> !(mcpItem instanceof ProposedItem));
     Stream<? extends BatchItem> combinedChangeItems =
@@ -147,6 +147,17 @@ private Stream<? extends BatchItem> proposedItemsToChangeItemStream(List<MCPItem
     return Stream.concat(combinedChangeItems, sideEffectItems);
   }
 
+  private static BatchItem patchDiscriminator(MCPItem mcpItem, AspectRetriever aspectRetriever) {
+    if (ChangeType.PATCH.equals(mcpItem.getChangeType())) {
+      return PatchItemImpl.PatchItemImplBuilder.build(
+          mcpItem.getMetadataChangeProposal(),
+          mcpItem.getAuditStamp(),
+          aspectRetriever.getEntityRegistry());
+    }
+    return ChangeItemImpl.ChangeItemImplBuilder.build(
+        mcpItem.getMetadataChangeProposal(), mcpItem.getAuditStamp(), aspectRetriever);
+  }
+
   public static class AspectsBatchImplBuilder {
     /**
      * Just one aspect record template
@@ -164,6 +175,14 @@ public AspectsBatchImplBuilder mcps(
         Collection<MetadataChangeProposal> mcps,
         AuditStamp auditStamp,
         RetrieverContext retrieverContext) {
+      return mcps(mcps, auditStamp, retrieverContext, false);
+    }
+
+    public AspectsBatchImplBuilder mcps(
+        Collection<MetadataChangeProposal> mcps,
+        AuditStamp auditStamp,
+        RetrieverContext retrieverContext,
+        boolean alternateMCPValidation) {
 
       retrieverContext(retrieverContext);
       items(
@@ -171,6 +190,18 @@ public AspectsBatchImplBuilder mcps(
               .map(
                   mcp -> {
                     try {
+                      if (alternateMCPValidation) {
+                        EntitySpec entitySpec =
+                            retrieverContext
+                                .getAspectRetriever()
+                                .getEntityRegistry()
+                                .getEntitySpec(mcp.getEntityType());
+                        return ProposedItem.builder()
+                            .metadataChangeProposal(mcp)
+                            .entitySpec(entitySpec)
+                            .auditStamp(auditStamp)
+                            .build();
+                      }
                       if (mcp.getChangeType().equals(ChangeType.PATCH)) {
                         return PatchItemImpl.PatchItemImplBuilder.build(
                             mcp,
@@ -190,14 +221,32 @@ public AspectsBatchImplBuilder mcps(
       return this;
     }
 
+    private static <T extends BatchItem> List<T> filterRepeats(Collection<T> items) {
+      List<T> result = new ArrayList<>();
+      Map<Pair<Urn, String>, T> last = new HashMap<>();
+
+      for (T item : items) {
+        Pair<Urn, String> urnAspect = Pair.of(item.getUrn(), item.getAspectName());
+        // Check if this item is a duplicate of the previous
+        if (!last.containsKey(urnAspect) || !item.isDatabaseDuplicateOf(last.get(urnAspect))) {
+          result.add(item);
+        }
+        last.put(urnAspect, item);
+      }
+
+      return result;
+    }
+
     public AspectsBatchImpl build() {
+      this.nonRepeatedItems = filterRepeats(this.items);
+
       ValidationExceptionCollection exceptions =
-          AspectsBatch.validateProposed(this.items, this.retrieverContext);
+          AspectsBatch.validateProposed(this.nonRepeatedItems, this.retrieverContext);
       if (!exceptions.isEmpty()) {
         throw new IllegalArgumentException("Failed to validate MCP due to: " + exceptions);
       }
 
-      return new AspectsBatchImpl(this.items, this.retrieverContext);
+      return new AspectsBatchImpl(this.items, this.nonRepeatedItems, this.retrieverContext);
     }
   }
 
diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java
index 6f45a36d1daf4..64263859e4aad 100644
--- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java
+++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java
@@ -3,11 +3,13 @@
 import com.datahub.util.exception.ModelConversionException;
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
+import com.linkedin.data.template.DataTemplateUtil;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.data.template.StringMap;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.aspect.SystemAspect;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.ChangeMCP;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate;
@@ -269,6 +271,11 @@ private static RecordTemplate convertToRecordTemplate(
     }
   }
 
+  @Override
+  public boolean isDatabaseDuplicateOf(BatchItem other) {
+    return equals(other);
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -280,13 +287,15 @@ public boolean equals(Object o) {
     ChangeItemImpl that = (ChangeItemImpl) o;
     return urn.equals(that.urn)
         && aspectName.equals(that.aspectName)
+        && changeType.equals(that.changeType)
         && Objects.equals(systemMetadata, that.systemMetadata)
-        && recordTemplate.equals(that.recordTemplate);
+        && Objects.equals(auditStamp, that.auditStamp)
+        && DataTemplateUtil.areEqual(recordTemplate, that.recordTemplate);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(urn, aspectName, systemMetadata, recordTemplate);
+    return Objects.hash(urn, aspectName, changeType, systemMetadata, auditStamp, recordTemplate);
   }
 
   @Override
diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java
index 9c1ded284fa0b..40bcb0fa8ed2d 100644
--- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java
+++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java
@@ -6,6 +6,7 @@
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.aspect.SystemAspect;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.ChangeMCP;
 import com.linkedin.metadata.entity.EntityApiUtils;
 import com.linkedin.metadata.entity.EntityAspect;
@@ -115,6 +116,11 @@ public DeleteItemImpl build(AspectRetriever aspectRetriever) {
     }
   }
 
+  @Override
+  public boolean isDatabaseDuplicateOf(BatchItem other) {
+    return equals(other);
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java
index a5afd4651ed2c..85923a28a64be 100644
--- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java
+++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java
@@ -5,6 +5,7 @@
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.MCLItem;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.entity.AspectUtils;
@@ -158,6 +159,11 @@ private static Pair<RecordTemplate, RecordTemplate> convertToRecordTemplate(
     }
   }
 
+  @Override
+  public boolean isDatabaseDuplicateOf(BatchItem other) {
+    return equals(other);
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java
index 43a7d00248a22..2543d99ac6af3 100644
--- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java
+++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java
@@ -14,6 +14,7 @@
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.batch.PatchMCP;
 import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine;
@@ -203,7 +204,7 @@ public static PatchItemImpl build(
           .build(entityRegistry);
     }
 
-    private static JsonPatch convertToJsonPatch(MetadataChangeProposal mcp) {
+    public static JsonPatch convertToJsonPatch(MetadataChangeProposal mcp) {
       JsonNode json;
       try {
         return Json.createPatch(
@@ -216,6 +217,11 @@ private static JsonPatch convertToJsonPatch(MetadataChangeProposal mcp) {
     }
   }
 
+  @Override
+  public boolean isDatabaseDuplicateOf(BatchItem other) {
+    return equals(other);
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -228,12 +234,13 @@ public boolean equals(Object o) {
     return urn.equals(that.urn)
         && aspectName.equals(that.aspectName)
         && Objects.equals(systemMetadata, that.systemMetadata)
+        && auditStamp.equals(that.auditStamp)
         && patch.equals(that.patch);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(urn, aspectName, systemMetadata, patch);
+    return Objects.hash(urn, aspectName, systemMetadata, auditStamp, patch);
   }
 
   @Override
diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ProposedItem.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ProposedItem.java
index 132a731d278af..370f1f6f073e6 100644
--- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ProposedItem.java
+++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ProposedItem.java
@@ -4,13 +4,16 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.utils.EntityKeyUtils;
 import com.linkedin.metadata.utils.GenericRecordUtils;
+import com.linkedin.metadata.utils.SystemMetadataUtils;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.mxe.SystemMetadata;
+import java.util.Objects;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.Builder;
@@ -83,4 +86,44 @@ public SystemMetadata getSystemMetadata() {
   public ChangeType getChangeType() {
     return metadataChangeProposal.getChangeType();
   }
+
+  @Override
+  public boolean isDatabaseDuplicateOf(BatchItem other) {
+    return equals(other);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    ProposedItem that = (ProposedItem) o;
+    return metadataChangeProposal.equals(that.metadataChangeProposal)
+        && auditStamp.equals(that.auditStamp);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = metadataChangeProposal.hashCode();
+    result = 31 * result + auditStamp.hashCode();
+    return result;
+  }
+
+  public static class ProposedItemBuilder {
+    public ProposedItem build() {
+      // Ensure systemMetadata
+      return new ProposedItem(
+          Objects.requireNonNull(this.metadataChangeProposal)
+              .setSystemMetadata(
+                  SystemMetadataUtils.generateSystemMetadataIfEmpty(
+                      this.metadataChangeProposal.getSystemMetadata())),
+          this.auditStamp,
+          this.entitySpec,
+          this.aspectSpec);
+    }
+  }
 }
diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java
index f7e639ecf3603..5e1f09fcc6439 100644
--- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java
+++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java
@@ -1,6 +1,7 @@
 package com.linkedin.metadata.entity.validation;
 
 import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.schema.validation.ValidationResult;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.metadata.Constants;
@@ -10,16 +11,27 @@
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import java.net.URISyntaxException;
+import java.net.URLDecoder;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Set;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public class ValidationApiUtils {
+  public static final String STRICT_URN_VALIDATION_ENABLED = "STRICT_URN_VALIDATION_ENABLED";
   public static final int URN_NUM_BYTES_LIMIT = 512;
+  // Related to BrowsePathv2
   public static final String URN_DELIMITER_SEPARATOR = "␟";
+  // https://datahubproject.io/docs/what/urn/#restrictions
+  public static final Set<String> ILLEGAL_URN_COMPONENT_CHARACTERS = Set.of("(", ")");
+  public static final Set<String> ILLEGAL_URN_TUPLE_CHARACTERS = Set.of(",");
 
   /**
    * Validates a {@link RecordTemplate} and throws {@link ValidationException} if validation fails.
@@ -38,6 +50,16 @@ public static void validateOrThrow(RecordTemplate record) {
   }
 
   public static void validateUrn(@Nonnull EntityRegistry entityRegistry, @Nonnull final Urn urn) {
+    validateUrn(
+        entityRegistry,
+        urn,
+        Boolean.TRUE.equals(
+            Boolean.parseBoolean(
+                System.getenv().getOrDefault(STRICT_URN_VALIDATION_ENABLED, "false"))));
+  }
+
+  public static void validateUrn(
+      @Nonnull EntityRegistry entityRegistry, @Nonnull final Urn urn, boolean strict) {
     EntityRegistryUrnValidator validator = new EntityRegistryUrnValidator(entityRegistry);
     validator.setCurrentEntitySpec(entityRegistry.getEntitySpec(urn.getEntityType()));
     RecordTemplateValidator.validate(
@@ -59,10 +81,31 @@ public static void validateUrn(@Nonnull EntityRegistry entityRegistry, @Nonnull
               + Integer.toString(URN_NUM_BYTES_LIMIT)
               + " bytes (when URL encoded)");
     }
+
     if (urn.toString().contains(URN_DELIMITER_SEPARATOR)) {
       throw new IllegalArgumentException(
           "Error: URN cannot contain " + URN_DELIMITER_SEPARATOR + " character");
     }
+
+    int totalParts = urn.getEntityKey().getParts().size();
+    List<String> illegalComponents =
+        urn.getEntityKey().getParts().stream()
+            .flatMap(part -> processUrnPartRecursively(part, totalParts))
+            .collect(Collectors.toList());
+
+    if (!illegalComponents.isEmpty()) {
+      String message =
+          String.format(
+              "Illegal `%s` characters detected in URN %s component(s): %s",
+              ILLEGAL_URN_COMPONENT_CHARACTERS, urn, illegalComponents);
+
+      if (strict) {
+        throw new IllegalArgumentException(message);
+      } else {
+        log.error(message);
+      }
+    }
+
     try {
       Urn.createFromString(urn.toString());
     } catch (URISyntaxException e) {
@@ -70,6 +113,28 @@ public static void validateUrn(@Nonnull EntityRegistry entityRegistry, @Nonnull
     }
   }
 
+  /** Recursively process URN parts with URL decoding */
+  private static Stream<String> processUrnPartRecursively(String urnPart, int totalParts) {
+    String decodedPart =
+        URLDecoder.decode(URLEncodingFixer.fixURLEncoding(urnPart), StandardCharsets.UTF_8);
+    if (decodedPart.startsWith("urn:li:")) {
+      // Recursively process nested URN after decoding
+      int nestedParts = UrnUtils.getUrn(decodedPart).getEntityKey().getParts().size();
+      return UrnUtils.getUrn(decodedPart).getEntityKey().getParts().stream()
+          .flatMap(part -> processUrnPartRecursively(part, nestedParts));
+    }
+    if (totalParts > 1) {
+      if (ILLEGAL_URN_TUPLE_CHARACTERS.stream().anyMatch(c -> urnPart.contains(c))) {
+        return Stream.of(urnPart);
+      }
+    }
+    if (ILLEGAL_URN_COMPONENT_CHARACTERS.stream().anyMatch(c -> urnPart.contains(c))) {
+      return Stream.of(urnPart);
+    }
+
+    return Stream.empty();
+  }
+
   /**
    * Validates a {@link RecordTemplate} and logs a warning if validation fails.
    *
@@ -123,4 +188,53 @@ public static void validateRecordTemplate(
       RecordTemplateValidator.validate(aspect, resultFunction, validator);
     }
   }
+
+  /**
+   * Fixes malformed URL encoding by escaping unescaped % characters while preserving valid
+   * percent-encoded sequences.
+   */
+  private static class URLEncodingFixer {
+    /**
+     * @param input The potentially malformed URL-encoded string
+     * @return A string with proper URL encoding that can be safely decoded
+     */
+    public static String fixURLEncoding(String input) {
+      if (input == null) {
+        return null;
+      }
+
+      StringBuilder result = new StringBuilder(input.length() * 2);
+      int i = 0;
+
+      while (i < input.length()) {
+        char currentChar = input.charAt(i);
+
+        if (currentChar == '%') {
+          if (i + 2 < input.length()) {
+            // Check if the next two characters form a valid hex pair
+            String hexPair = input.substring(i + 1, i + 3);
+            if (isValidHexPair(hexPair)) {
+              // This is a valid percent-encoded sequence, keep it as is
+              result.append(currentChar);
+            } else {
+              // Invalid sequence, escape the % character
+              result.append("%25");
+            }
+          } else {
+            // % at the end of string, escape it
+            result.append("%25");
+          }
+        } else {
+          result.append(currentChar);
+        }
+        i++;
+      }
+
+      return result.toString();
+    }
+
+    private static boolean isValidHexPair(String pair) {
+      return pair.matches("[0-9A-Fa-f]{2}");
+    }
+  }
 }
diff --git a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java
index 31dd868b4cb4a..a3099b9ee21ea 100644
--- a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java
+++ b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java
@@ -6,6 +6,7 @@
 import static org.testng.Assert.assertEquals;
 
 import com.google.common.collect.ImmutableList;
+import com.linkedin.common.AuditStamp;
 import com.linkedin.common.FabricType;
 import com.linkedin.common.Status;
 import com.linkedin.common.urn.DataPlatformUrn;
@@ -15,7 +16,7 @@
 import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
 import com.linkedin.dataset.DatasetProperties;
 import com.linkedin.events.metadata.ChangeType;
-import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.patch.GenericJsonPatch;
@@ -41,6 +42,7 @@
 import io.datahubproject.metadata.context.RetrieverContext;
 import java.nio.charset.StandardCharsets;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -54,7 +56,7 @@
 
 public class AspectsBatchImplTest {
   private EntityRegistry testRegistry;
-  private AspectRetriever mockAspectRetriever;
+  private CachingAspectRetriever mockAspectRetriever;
   private RetrieverContext retrieverContext;
 
   @BeforeTest
@@ -73,12 +75,12 @@ public void beforeTest() throws EntityRegistryException {
 
   @BeforeMethod
   public void setup() {
-    this.mockAspectRetriever = mock(AspectRetriever.class);
+    this.mockAspectRetriever = mock(CachingAspectRetriever.class);
     when(this.mockAspectRetriever.getEntityRegistry()).thenReturn(testRegistry);
     this.retrieverContext =
         RetrieverContext.builder()
             .searchRetriever(mock(SearchRetriever.class))
-            .aspectRetriever(mockAspectRetriever)
+            .cachingAspectRetriever(mockAspectRetriever)
             .graphRetriever(mock(GraphRetriever.class))
             .build();
   }
@@ -120,7 +122,7 @@ public void toUpsertBatchItemsChangeItemTest() {
         AspectsBatchImpl.builder().items(testItems).retrieverContext(retrieverContext).build();
 
     assertEquals(
-        testBatch.toUpsertBatchItems(Map.of()),
+        testBatch.toUpsertBatchItems(new HashMap<>(), new HashMap<>()),
         Pair.of(Map.of(), testItems),
         "Expected noop, pass through with no additional MCPs or changes");
   }
@@ -176,7 +178,7 @@ public void toUpsertBatchItemsPatchItemTest() {
         AspectsBatchImpl.builder().items(testItems).retrieverContext(retrieverContext).build();
 
     assertEquals(
-        testBatch.toUpsertBatchItems(Map.of()),
+        testBatch.toUpsertBatchItems(new HashMap<>(), new HashMap<>()),
         Pair.of(
             Map.of(),
             List.of(
@@ -195,7 +197,7 @@ public void toUpsertBatchItemsPatchItemTest() {
                     .recordTemplate(
                         new StructuredProperties()
                             .setProperties(new StructuredPropertyValueAssignmentArray()))
-                    .systemMetadata(testItems.get(0).getSystemMetadata())
+                    .systemMetadata(testItems.get(0).getSystemMetadata().setVersion("1"))
                     .build(mockAspectRetriever),
                 ChangeItemImpl.builder()
                     .urn(
@@ -212,13 +214,14 @@ public void toUpsertBatchItemsPatchItemTest() {
                     .recordTemplate(
                         new StructuredProperties()
                             .setProperties(new StructuredPropertyValueAssignmentArray()))
-                    .systemMetadata(testItems.get(1).getSystemMetadata())
+                    .systemMetadata(testItems.get(1).getSystemMetadata().setVersion("1"))
                     .build(mockAspectRetriever))),
         "Expected patch items converted to upsert change items");
   }
 
   @Test
   public void toUpsertBatchItemsProposedItemTest() {
+    AuditStamp auditStamp = AuditStampUtils.createDefaultAuditStamp();
     List<ProposedItem> testItems =
         List.of(
             ProposedItem.builder()
@@ -238,7 +241,7 @@ public void toUpsertBatchItemsProposedItemTest() {
                                     ByteString.copyString(
                                         "{\"foo\":\"bar\"}", StandardCharsets.UTF_8)))
                         .setSystemMetadata(new SystemMetadata()))
-                .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+                .auditStamp(auditStamp)
                 .build(),
             ProposedItem.builder()
                 .entitySpec(testRegistry.getEntitySpec(DATASET_ENTITY_NAME))
@@ -257,14 +260,14 @@ public void toUpsertBatchItemsProposedItemTest() {
                                     ByteString.copyString(
                                         "{\"foo\":\"bar\"}", StandardCharsets.UTF_8)))
                         .setSystemMetadata(new SystemMetadata()))
-                .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+                .auditStamp(auditStamp)
                 .build());
 
     AspectsBatchImpl testBatch =
         AspectsBatchImpl.builder().items(testItems).retrieverContext(retrieverContext).build();
 
     assertEquals(
-        testBatch.toUpsertBatchItems(Map.of()),
+        testBatch.toUpsertBatchItems(new HashMap<>(), new HashMap<>()),
         Pair.of(
             Map.of(),
             List.of(
@@ -279,8 +282,8 @@ public void toUpsertBatchItemsProposedItemTest() {
                         testRegistry
                             .getEntitySpec(DATASET_ENTITY_NAME)
                             .getAspectSpec(STATUS_ASPECT_NAME))
-                    .auditStamp(AuditStampUtils.createDefaultAuditStamp())
-                    .systemMetadata(testItems.get(0).getSystemMetadata())
+                    .auditStamp(auditStamp)
+                    .systemMetadata(testItems.get(0).getSystemMetadata().setVersion("1"))
                     .recordTemplate(new Status().setRemoved(false))
                     .build(mockAspectRetriever),
                 ChangeItemImpl.builder()
@@ -294,8 +297,8 @@ public void toUpsertBatchItemsProposedItemTest() {
                         testRegistry
                             .getEntitySpec(DATASET_ENTITY_NAME)
                             .getAspectSpec(STATUS_ASPECT_NAME))
-                    .auditStamp(AuditStampUtils.createDefaultAuditStamp())
-                    .systemMetadata(testItems.get(1).getSystemMetadata())
+                    .auditStamp(auditStamp)
+                    .systemMetadata(testItems.get(1).getSystemMetadata().setVersion("1"))
                     .recordTemplate(new Status().setRemoved(false))
                     .build(mockAspectRetriever))),
         "Mutation to status aspect");
@@ -328,7 +331,7 @@ public void singleInvalidDoesntBreakBatch() {
             .build();
 
     assertEquals(
-        testBatch.toUpsertBatchItems(Map.of()).getSecond().size(),
+        testBatch.toUpsertBatchItems(new HashMap<>(), new HashMap<>()).getSecond().size(),
         1,
         "Expected 1 valid mcp to be passed through.");
   }
diff --git a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java
new file mode 100644
index 0000000000000..2ab6a50945ba3
--- /dev/null
+++ b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java
@@ -0,0 +1,162 @@
+package com.linkedin.metadata.entity.validation;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
+import java.net.URISyntaxException;
+import org.testng.annotations.Test;
+
+public class ValidationApiUtilsTest {
+  private static final EntityRegistry entityRegistry =
+      TestOperationContexts.defaultEntityRegistry();
+
+  @Test
+  public void testValidateDatasetUrn() {
+    Urn validUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,/path/to/data,PROD)");
+    ValidationApiUtils.validateUrn(entityRegistry, validUrn, true);
+    // If no exception is thrown, test passes
+  }
+
+  @Test
+  public void testSimpleUrnColon() {
+    ValidationApiUtils.validateUrn(
+        entityRegistry, UrnUtils.getUrn("urn:li:corpuser:foo:bar"), true);
+    ValidationApiUtils.validateUrn(
+        entityRegistry, UrnUtils.getUrn("urn:li:dataPlatform:abc:def"), true);
+    ValidationApiUtils.validateUrn(
+        entityRegistry, UrnUtils.getUrn("urn:li:corpuser:foo:bar@example.com"), true);
+    // If no exception is thrown, test passes
+  }
+
+  @Test
+  public void testSimpleUrnComma() {
+    ValidationApiUtils.validateUrn(entityRegistry, UrnUtils.getUrn("urn:li:corpuser:,"), true);
+    // If no exception is thrown, test passes
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testTupleUrnComma() {
+    ValidationApiUtils.validateUrn(
+        entityRegistry, UrnUtils.getUrn("urn:li:dashboard:(looker,dashboards,thelook)"), true);
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testFabricTypeCasing() {
+    // prod != PROD
+    ValidationApiUtils.validateUrn(
+        entityRegistry,
+        UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:abc:def,table_name,prod)"),
+        true);
+  }
+
+  @Test
+  public void testComplexUrnColon() throws URISyntaxException {
+    Urn validUrn =
+        Urn.createFromString(
+            "urn:li:dataset:(urn:li:dataPlatform:s3,urn:li:dataset:%28urn:li:dataPlatform:s3%2Ctest-datalake-concepts/prog_maintenance%2CPROD%29,PROD)");
+    ValidationApiUtils.validateUrn(entityRegistry, validUrn, true);
+    // If no exception is thrown, test passes
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testFabricTypeParen() {
+    Urn invalidUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,/path/to/data,())");
+    ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true);
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testUrnWithTrailingWhitespace() {
+    Urn invalidUrn =
+        UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,/path/to/data,PROD) ");
+    ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true);
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testUrnWithIllegalDelimiter() {
+    Urn invalidUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs␟path,PROD)");
+    ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true);
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testComplexUrnWithParens1() {
+    Urn invalidUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,(illegal),PROD)");
+    ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true);
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testComplexUrnWithParens2() {
+    Urn invalidUrn =
+        UrnUtils.getUrn(
+            "urn:li:dataJob:(urn:li:dataFlow:(mssql,1/2/3/4.c_n on %28LOCAL%29,PROD),1/2/3/4.c_n on (LOCAL))");
+    ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true);
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testSimpleUrnWithParens() {
+    Urn invalidUrn = UrnUtils.getUrn("urn:li:corpuser:(foo)123");
+    ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true);
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testExcessiveLength() {
+    StringBuilder longPath = new StringBuilder("urn:li:dataset:(urn:li:dataPlatform:hdfs,");
+    // Create a path that will exceed 512 bytes when URL encoded
+    for (int i = 0; i < 500; i++) {
+      longPath.append("very/long/path/");
+    }
+    longPath.append(",PROD)");
+    Urn invalidUrn = UrnUtils.getUrn(longPath.toString());
+
+    ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true);
+  }
+
+  @Test
+  public void testValidComplexUrn() {
+    Urn validUrn =
+        UrnUtils.getUrn(
+            "urn:li:dataset:(urn:li:dataPlatform:bigquery,myproject.dataset.table,PROD)");
+
+    ValidationApiUtils.validateUrn(entityRegistry, validUrn, true);
+    // If no exception is thrown, test passes
+  }
+
+  @Test(expectedExceptions = NullPointerException.class)
+  public void testUrnNull() {
+    ValidationApiUtils.validateUrn(entityRegistry, null, true);
+  }
+
+  @Test
+  public void testValidPartialUrlEncode() {
+    Urn validUrn = UrnUtils.getUrn("urn:li:assertion:123=-%28__% weekly__%29");
+
+    ValidationApiUtils.validateUrn(entityRegistry, validUrn, true);
+    // If no exception is thrown, test passes
+  }
+
+  @Test
+  public void testValidPartialUrlEncode2() {
+    Urn validUrn =
+        UrnUtils.getUrn(
+            "urn:li:dataset:(urn:li:dataPlatform:s3,urn:li:dataset:%28urn:li:dataPlatform:s3%2Ctest-datalake-concepts%prog_maintenance%2CPROD%29,PROD)");
+
+    ValidationApiUtils.validateUrn(entityRegistry, validUrn, true);
+    // If no exception is thrown, test passes
+  }
+
+  @Test
+  public void testValidColon() {
+    Urn validUrn =
+        UrnUtils.getUrn("urn:li:dashboard:(looker,dashboards.thelook::cohort_data_tool)");
+
+    ValidationApiUtils.validateUrn(entityRegistry, validUrn, true);
+    // If no exception is thrown, test passes
+  }
+
+  @Test
+  public void testNoTupleComma() {
+    Urn invalidUrn = UrnUtils.getUrn("urn:li:corpuser:,");
+    ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true);
+    // If no exception is thrown, test passes
+  }
+}
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/aspect/hooks/IgnoreUnknownMutator.java b/metadata-io/src/main/java/com/linkedin/metadata/aspect/hooks/IgnoreUnknownMutator.java
index 8d6bdffceacb9..f5cc421042e36 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/aspect/hooks/IgnoreUnknownMutator.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/aspect/hooks/IgnoreUnknownMutator.java
@@ -1,8 +1,14 @@
 package com.linkedin.metadata.aspect.hooks;
 
+import static com.linkedin.events.metadata.ChangeType.CREATE;
+import static com.linkedin.events.metadata.ChangeType.CREATE_ENTITY;
+import static com.linkedin.events.metadata.ChangeType.UPDATE;
+import static com.linkedin.events.metadata.ChangeType.UPSERT;
+
 import com.datahub.util.exception.ModelConversionException;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.data.transform.filter.request.MaskTree;
+import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.RetrieverContext;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
@@ -14,6 +20,7 @@
 import com.linkedin.mxe.GenericAspect;
 import com.linkedin.restli.internal.server.util.RestUtils;
 import java.util.Collection;
+import java.util.Set;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.Getter;
@@ -27,6 +34,11 @@
 @Getter
 @Accessors(chain = true)
 public class IgnoreUnknownMutator extends MutationHook {
+  private static final Set<String> SUPPORTED_MIME_TYPES =
+      Set.of("application/json", "application/json-patch+json");
+  private static final Set<ChangeType> MUTATION_TYPES =
+      Set.of(CREATE, CREATE_ENTITY, UPSERT, UPDATE);
+
   @Nonnull private AspectPluginConfig config;
 
   @Override
@@ -42,8 +54,8 @@ protected Stream<MCPItem> proposalMutation(
                     item.getAspectSpec().getName());
                 return false;
               }
-              if (!"application/json"
-                  .equals(item.getMetadataChangeProposal().getAspect().getContentType())) {
+              if (!SUPPORTED_MIME_TYPES.contains(
+                  item.getMetadataChangeProposal().getAspect().getContentType())) {
                 log.warn(
                     "Dropping unknown content type {} for aspect {} on entity {}",
                     item.getMetadataChangeProposal().getAspect().getContentType(),
@@ -55,25 +67,27 @@ protected Stream<MCPItem> proposalMutation(
             })
         .peek(
             item -> {
-              try {
-                AspectSpec aspectSpec = item.getEntitySpec().getAspectSpec(item.getAspectName());
-                GenericAspect aspect = item.getMetadataChangeProposal().getAspect();
-                RecordTemplate recordTemplate =
-                    GenericRecordUtils.deserializeAspect(
-                        aspect.getValue(), aspect.getContentType(), aspectSpec);
+              if (MUTATION_TYPES.contains(item.getChangeType())) {
                 try {
-                  ValidationApiUtils.validateOrThrow(recordTemplate);
-                } catch (ValidationException | ModelConversionException e) {
-                  log.warn(
-                      "Failed to validate aspect. Coercing aspect {} on entity {}",
-                      item.getAspectName(),
-                      item.getEntitySpec().getName());
-                  RestUtils.trimRecordTemplate(recordTemplate, new MaskTree(), false);
-                  item.getMetadataChangeProposal()
-                      .setAspect(GenericRecordUtils.serializeAspect(recordTemplate));
+                  AspectSpec aspectSpec = item.getEntitySpec().getAspectSpec(item.getAspectName());
+                  GenericAspect aspect = item.getMetadataChangeProposal().getAspect();
+                  RecordTemplate recordTemplate =
+                      GenericRecordUtils.deserializeAspect(
+                          aspect.getValue(), aspect.getContentType(), aspectSpec);
+                  try {
+                    ValidationApiUtils.validateOrThrow(recordTemplate);
+                  } catch (ValidationException | ModelConversionException e) {
+                    log.warn(
+                        "Failed to validate aspect. Coercing aspect {} on entity {}",
+                        item.getAspectName(),
+                        item.getEntitySpec().getName());
+                    RestUtils.trimRecordTemplate(recordTemplate, new MaskTree(), false);
+                    item.getMetadataChangeProposal()
+                        .setAspect(GenericRecordUtils.serializeAspect(recordTemplate));
+                  }
+                } catch (Exception e) {
+                  throw new RuntimeException(e);
                 }
-              } catch (Exception e) {
-                throw new RuntimeException(e);
               }
             });
   }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java b/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java
index a4b2e991b6e1e..82bc0ae1409c5 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java
@@ -14,6 +14,7 @@
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.data.template.SetMode;
 import com.linkedin.data.template.StringArray;
+import com.linkedin.data.template.StringMap;
 import com.linkedin.dataplatform.DataPlatformInfo;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.events.metadata.ChangeType;
@@ -21,6 +22,7 @@
 import com.linkedin.metadata.aspect.batch.AspectsBatch;
 import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.MCPItem;
+import com.linkedin.metadata.aspect.validation.CreateIfNotExistsValidator;
 import com.linkedin.metadata.entity.EntityApiUtils;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
@@ -98,7 +100,8 @@ public static List<MCPItem> getAdditionalChanges(
             .filter(item -> SUPPORTED_TYPES.contains(item.getChangeType()))
             .collect(Collectors.groupingBy(BatchItem::getUrn));
 
-    Set<Urn> urnsWithExistingKeyAspects = entityService.exists(opContext, itemsByUrn.keySet());
+    Set<Urn> urnsWithExistingKeyAspects =
+        entityService.exists(opContext, itemsByUrn.keySet(), true, true);
 
     // create default aspects when key aspect is missing
     return itemsByUrn.entrySet().stream()
@@ -126,7 +129,7 @@ public static List<MCPItem> getAdditionalChanges(
               // pick the first item as a template (use entity information)
               MCPItem templateItem = aspectsEntry.getValue().get(0);
 
-              // generate default aspects (including key aspect, always upserts)
+              // generate default aspects (including key aspect)
               return defaultAspects.stream()
                   .map(
                       entry ->
@@ -134,7 +137,7 @@ public static List<MCPItem> getAdditionalChanges(
                               getProposalFromAspectForDefault(
                                   entry.getKey(), entry.getValue(), entityKeyAspect, templateItem),
                               templateItem.getAuditStamp(),
-                              opContext.getAspectRetrieverOpt().get()))
+                              opContext.getAspectRetriever()))
                   .filter(Objects::nonNull);
             })
         .collect(Collectors.toList());
@@ -215,7 +218,7 @@ private static List<Pair<String, RecordTemplate>> generateDefaultAspectsIfMissin
     if (!fetchAspects.isEmpty()) {
 
       Set<String> latestAspects =
-          entityService.getLatestAspectsForUrn(opContext, urn, fetchAspects).keySet();
+          entityService.getLatestAspectsForUrn(opContext, urn, fetchAspects, true).keySet();
 
       return fetchAspects.stream()
           .filter(aspectName -> !latestAspects.contains(aspectName))
@@ -347,6 +350,11 @@ public static MetadataChangeProposal getProposalFromAspectForDefault(
     proposal.setAspectName(aspectName);
     // already checked existence, default aspects should be changeType CREATE
     proposal.setChangeType(ChangeType.CREATE);
+    proposal.setHeaders(
+        new StringMap(
+            Map.of(
+                CreateIfNotExistsValidator.FILTER_EXCEPTION_HEADER,
+                CreateIfNotExistsValidator.FILTER_EXCEPTION_VALUE)));
 
     // Set fields determined from original
     if (templateItem.getSystemMetadata() != null) {
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidator.java b/metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidator.java
new file mode 100644
index 0000000000000..b77d3b48d5bd5
--- /dev/null
+++ b/metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidator.java
@@ -0,0 +1,70 @@
+package com.linkedin.metadata.aspect.validation;
+
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_ABORTED;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_CANCELLED;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_DUPLICATE;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_SUCCESS;
+
+import com.linkedin.execution.ExecutionRequestResult;
+import com.linkedin.metadata.aspect.RetrieverContext;
+import com.linkedin.metadata.aspect.batch.BatchItem;
+import com.linkedin.metadata.aspect.batch.ChangeMCP;
+import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
+import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
+import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+
+/** A Validator for StructuredProperties Aspect that is attached to entities like Datasets, etc. */
+@Setter
+@Getter
+@Slf4j
+@Accessors(chain = true)
+public class ExecutionRequestResultValidator extends AspectPayloadValidator {
+  private static final Set<String> IMMUTABLE_STATUS =
+      Set.of(
+          EXECUTION_REQUEST_STATUS_ABORTED,
+          EXECUTION_REQUEST_STATUS_CANCELLED,
+          EXECUTION_REQUEST_STATUS_SUCCESS,
+          EXECUTION_REQUEST_STATUS_DUPLICATE);
+
+  @Nonnull private AspectPluginConfig config;
+
+  @Override
+  protected Stream<AspectValidationException> validateProposedAspects(
+      @Nonnull Collection<? extends BatchItem> mcpItems,
+      @Nonnull RetrieverContext retrieverContext) {
+    return Stream.of();
+  }
+
+  @Override
+  protected Stream<AspectValidationException> validatePreCommitAspects(
+      @Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull RetrieverContext retrieverContext) {
+    return changeMCPs.stream()
+        .filter(item -> item.getPreviousRecordTemplate() != null)
+        .map(
+            item -> {
+              ExecutionRequestResult existingResult =
+                  item.getPreviousAspect(ExecutionRequestResult.class);
+
+              if (IMMUTABLE_STATUS.contains(existingResult.getStatus())) {
+                ExecutionRequestResult currentResult = item.getAspect(ExecutionRequestResult.class);
+                return AspectValidationException.forItem(
+                    item,
+                    String.format(
+                        "Invalid update to immutable state for aspect dataHubExecutionRequestResult. Execution urn: %s previous status: %s. Denied status update: %s",
+                        item.getUrn(), existingResult.getStatus(), currentResult.getStatus()));
+              }
+
+              return null;
+            })
+        .filter(Objects::nonNull);
+  }
+}
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java b/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java
index bba8324d0c561..669ec751f87c6 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java
@@ -35,7 +35,7 @@ public EntityRegistry getEntityRegistry() {
   @Override
   public Aspect getLatestAspectObject(@Nonnull Urn urn, @Nonnull String aspectName) {
     try {
-      return entityClient.getLatestAspectObject(systemOperationContext, urn, aspectName);
+      return entityClient.getLatestAspectObject(systemOperationContext, urn, aspectName, false);
     } catch (RemoteInvocationException | URISyntaxException e) {
       throw new RuntimeException(e);
     }
@@ -49,7 +49,7 @@ public Map<Urn, Map<String, Aspect>> getLatestAspectObjects(
       return Map.of();
     } else {
       try {
-        return entityClient.getLatestAspects(systemOperationContext, urns, aspectNames);
+        return entityClient.getLatestAspects(systemOperationContext, urns, aspectNames, false);
       } catch (RemoteInvocationException | URISyntaxException e) {
         throw new RuntimeException(e);
       }
@@ -70,7 +70,8 @@ public Map<Urn, Map<String, SystemAspect>> getLatestSystemAspects(
             urnAspectNames.keySet(),
             urnAspectNames.values().stream()
                 .flatMap(Collection::stream)
-                .collect(Collectors.toSet()));
+                .collect(Collectors.toSet()),
+            false);
       } catch (RemoteInvocationException | URISyntaxException e) {
         throw new RuntimeException(e);
       }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java
index 60a991c19ae8b..3d35f5956b0f4 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java
@@ -18,12 +18,12 @@
 import com.linkedin.entity.Entity;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.aspect.EnvelopedAspect;
 import com.linkedin.metadata.aspect.EnvelopedAspectArray;
 import com.linkedin.metadata.aspect.VersionedAspect;
 import com.linkedin.metadata.aspect.batch.AspectsBatch;
-import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.browse.BrowseResult;
 import com.linkedin.metadata.browse.BrowseResultV2;
 import com.linkedin.metadata.entity.DeleteEntityService;
@@ -56,6 +56,7 @@
 import com.linkedin.parseq.retry.backoff.BackoffPolicy;
 import com.linkedin.parseq.retry.backoff.ExponentialBackoff;
 import com.linkedin.r2.RemoteInvocationException;
+import com.linkedin.util.Pair;
 import io.datahubproject.metadata.context.OperationContext;
 import io.opentelemetry.extension.annotations.WithSpan;
 import java.net.URISyntaxException;
@@ -97,7 +98,7 @@ public class JavaEntityClient implements EntityClient {
   private final TimeseriesAspectService timeseriesAspectService;
   private final RollbackService rollbackService;
   private final EventProducer eventProducer;
-  private final int batchGetV2Size;
+  private final EntityClientConfig entityClientConfig;
 
   @Override
   @Nullable
@@ -105,11 +106,17 @@ public EntityResponse getV2(
       @Nonnull OperationContext opContext,
       @Nonnull String entityName,
       @Nonnull final Urn urn,
-      @Nullable final Set<String> aspectNames)
+      @Nullable final Set<String> aspectNames,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException {
     final Set<String> projectedAspects =
         aspectNames == null ? opContext.getEntityAspectNames(entityName) : aspectNames;
-    return entityService.getEntityV2(opContext, entityName, urn, projectedAspects);
+    return entityService.getEntityV2(
+        opContext,
+        entityName,
+        urn,
+        projectedAspects,
+        alwaysIncludeKeyAspect == null || alwaysIncludeKeyAspect);
   }
 
   @Override
@@ -125,20 +132,25 @@ public Map<Urn, EntityResponse> batchGetV2(
       @Nonnull OperationContext opContext,
       @Nonnull String entityName,
       @Nonnull Set<Urn> urns,
-      @Nullable Set<String> aspectNames)
+      @Nullable Set<String> aspectNames,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException {
     final Set<String> projectedAspects =
         aspectNames == null ? opContext.getEntityAspectNames(entityName) : aspectNames;
 
     Map<Urn, EntityResponse> responseMap = new HashMap<>();
 
-    Iterators.partition(urns.iterator(), Math.max(1, batchGetV2Size))
+    Iterators.partition(urns.iterator(), Math.max(1, entityClientConfig.getBatchGetV2Size()))
         .forEachRemaining(
             batch -> {
               try {
                 responseMap.putAll(
                     entityService.getEntitiesV2(
-                        opContext, entityName, new HashSet<>(batch), projectedAspects));
+                        opContext,
+                        entityName,
+                        new HashSet<>(batch),
+                        projectedAspects,
+                        alwaysIncludeKeyAspect == null || alwaysIncludeKeyAspect));
               } catch (URISyntaxException e) {
                 throw new RuntimeException(e);
               }
@@ -159,7 +171,8 @@ public Map<Urn, EntityResponse> batchGetVersionedV2(
 
     Map<Urn, EntityResponse> responseMap = new HashMap<>();
 
-    Iterators.partition(versionedUrns.iterator(), Math.max(1, batchGetV2Size))
+    Iterators.partition(
+            versionedUrns.iterator(), Math.max(1, entityClientConfig.getBatchGetV2Size()))
         .forEachRemaining(
             batch -> {
               try {
@@ -760,42 +773,62 @@ public List<String> batchIngestProposals(
             : Constants.UNKNOWN_ACTOR;
     final AuditStamp auditStamp = AuditStampUtils.createAuditStamp(actorUrnStr);
 
-    AspectsBatch batch =
-        AspectsBatchImpl.builder()
-            .mcps(metadataChangeProposals, auditStamp, opContext.getRetrieverContext().get())
-            .build();
-
-    Map<BatchItem, List<IngestResult>> resultMap =
-        entityService.ingestProposal(opContext, batch, async).stream()
-            .collect(Collectors.groupingBy(IngestResult::getRequest));
-
-    // Update runIds
-    batch.getItems().stream()
-        .filter(resultMap::containsKey)
-        .forEach(
-            requestItem -> {
-              List<IngestResult> results = resultMap.get(requestItem);
-              Optional<Urn> resultUrn =
-                  results.stream().map(IngestResult::getUrn).filter(Objects::nonNull).findFirst();
-              resultUrn.ifPresent(
-                  urn -> tryIndexRunId(opContext, urn, requestItem.getSystemMetadata()));
+    List<String> updatedUrns = new ArrayList<>();
+    Iterators.partition(
+            metadataChangeProposals.iterator(), Math.max(1, entityClientConfig.getBatchGetV2Size()))
+        .forEachRemaining(
+            batch -> {
+              AspectsBatch aspectsBatch =
+                  AspectsBatchImpl.builder()
+                      .mcps(
+                          batch,
+                          auditStamp,
+                          opContext.getRetrieverContext(),
+                          opContext.getValidationContext().isAlternateValidation())
+                      .build();
+
+              List<IngestResult> results =
+                  entityService.ingestProposal(opContext, aspectsBatch, async);
+              entitySearchService.appendRunId(opContext, results);
+
+              Map<Pair<Urn, String>, List<IngestResult>> resultMap =
+                  results.stream()
+                      .collect(
+                          Collectors.groupingBy(
+                              result ->
+                                  Pair.of(
+                                      result.getRequest().getUrn(),
+                                      result.getRequest().getAspectName())));
+
+              // Preserve ordering
+              updatedUrns.addAll(
+                  aspectsBatch.getItems().stream()
+                      .map(
+                          requestItem -> {
+                            // Urns generated
+                            List<Urn> urnsForRequest =
+                                resultMap
+                                    .getOrDefault(
+                                        Pair.of(requestItem.getUrn(), requestItem.getAspectName()),
+                                        List.of())
+                                    .stream()
+                                    .map(IngestResult::getUrn)
+                                    .filter(Objects::nonNull)
+                                    .distinct()
+                                    .collect(Collectors.toList());
+
+                            // Update runIds
+                            urnsForRequest.forEach(
+                                urn ->
+                                    tryIndexRunId(opContext, urn, requestItem.getSystemMetadata()));
+
+                            return urnsForRequest.isEmpty()
+                                ? null
+                                : urnsForRequest.get(0).toString();
+                          })
+                      .collect(Collectors.toList()));
             });
-
-    // Preserve ordering
-    return batch.getItems().stream()
-        .map(
-            requestItem -> {
-              if (resultMap.containsKey(requestItem)) {
-                List<IngestResult> results = resultMap.get(requestItem);
-                return results.stream()
-                    .filter(r -> r.getUrn() != null)
-                    .findFirst()
-                    .map(r -> r.getUrn().toString())
-                    .orElse(null);
-              }
-              return null;
-            })
-        .collect(Collectors.toList());
+    return updatedUrns;
   }
 
   @SneakyThrows
@@ -861,8 +894,7 @@ public void rollbackIngestion(
   private void tryIndexRunId(
       @Nonnull OperationContext opContext, Urn entityUrn, @Nullable SystemMetadata systemMetadata) {
     if (systemMetadata != null && systemMetadata.hasRunId()) {
-      entitySearchService.appendRunId(
-          opContext, entityUrn.getEntityType(), entityUrn, systemMetadata.getRunId());
+      entitySearchService.appendRunId(opContext, entityUrn, systemMetadata.getRunId());
     }
   }
 
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java
index ab68abc69bce7..1d2fd422d7f46 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java
@@ -5,6 +5,7 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.client.EntityClientCache;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig;
 import com.linkedin.metadata.entity.DeleteEntityService;
@@ -43,7 +44,7 @@ public SystemJavaEntityClient(
       RollbackService rollbackService,
       EventProducer eventProducer,
       EntityClientCacheConfig cacheConfig,
-      int batchGetV2Size) {
+      EntityClientConfig entityClientConfig) {
     super(
         entityService,
         deleteEntityService,
@@ -54,7 +55,7 @@ public SystemJavaEntityClient(
         timeseriesAspectService,
         rollbackService,
         eventProducer,
-        batchGetV2Size);
+        entityClientConfig);
     this.operationContextMap = CacheBuilder.newBuilder().maximumSize(500).build();
     this.entityClientCache = buildEntityClientCache(SystemJavaEntityClient.class, cacheConfig);
   }
@@ -88,6 +89,6 @@ public Map<Urn, EntityResponse> batchGetV2NoCache(
       @Nonnull Set<Urn> urns,
       @Nullable Set<String> aspectNames)
       throws RemoteInvocationException, URISyntaxException {
-    return super.batchGetV2(opContext, entityName, urns, aspectNames);
+    return super.batchGetV2(opContext, entityName, urns, aspectNames, false);
   }
 }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java b/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java
index 544040d14f8b7..dae1a8ff51a2c 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java
@@ -9,6 +9,7 @@
 import com.linkedin.dataproduct.DataProductAssociation;
 import com.linkedin.dataproduct.DataProductAssociationArray;
 import com.linkedin.dataproduct.DataProductProperties;
+import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.RetrieverContext;
 import com.linkedin.metadata.aspect.batch.ChangeMCP;
 import com.linkedin.metadata.aspect.batch.MCLItem;
@@ -27,9 +28,11 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.Getter;
@@ -64,71 +67,108 @@ private static Stream<MCPItem> generatePatchRemove(
       MCLItem mclItem, @Nonnull RetrieverContext retrieverContext) {
 
     if (DATA_PRODUCT_PROPERTIES_ASPECT_NAME.equals(mclItem.getAspectName())) {
-      List<MCPItem> mcpItems = new ArrayList<>();
+
       DataProductProperties dataProductProperties = mclItem.getAspect(DataProductProperties.class);
       if (dataProductProperties == null) {
         log.error("Unable to process data product properties for urn: {}", mclItem.getUrn());
         return Stream.empty();
       }
-      for (DataProductAssociation dataProductAssociation :
+      DataProductAssociationArray newDataProductAssociationArray =
           Optional.ofNullable(dataProductProperties.getAssets())
-              .orElse(new DataProductAssociationArray())) {
-        RelatedEntitiesScrollResult result =
-            retrieverContext
-                .getGraphRetriever()
-                .scrollRelatedEntities(
-                    null,
-                    QueryUtils.newFilter(
-                        "urn", dataProductAssociation.getDestinationUrn().toString()),
-                    null,
-                    EMPTY_FILTER,
-                    ImmutableList.of("DataProductContains"),
-                    QueryUtils.newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING),
-                    Collections.emptyList(),
-                    null,
-                    10, // Should only ever be one, if ever greater than ten will decrease over time
-                    // to become consistent
-                    null,
-                    null);
-        if (!result.getEntities().isEmpty()) {
-          for (RelatedEntities entity : result.getEntities()) {
-            if (!mclItem.getUrn().equals(UrnUtils.getUrn(entity.getSourceUrn()))) {
-              EntitySpec entitySpec =
-                  retrieverContext
-                      .getAspectRetriever()
-                      .getEntityRegistry()
-                      .getEntitySpec(DATA_PRODUCT_ENTITY_NAME);
-              GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp();
-              patchOp.setOp(PatchOperationType.REMOVE.getValue());
-              patchOp.setPath(String.format("/assets/%s", entity.getDestinationUrn()));
-              mcpItems.add(
-                  PatchItemImpl.builder()
-                      .urn(UrnUtils.getUrn(entity.getSourceUrn()))
-                      .entitySpec(
-                          retrieverContext
-                              .getAspectRetriever()
-                              .getEntityRegistry()
-                              .getEntitySpec(DATA_PRODUCT_ENTITY_NAME))
-                      .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)
-                      .aspectSpec(entitySpec.getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME))
-                      .patch(
-                          GenericJsonPatch.builder()
-                              .arrayPrimaryKeys(
-                                  Map.of(
-                                      DataProductPropertiesTemplate.ASSETS_FIELD_NAME,
-                                      List.of(DataProductPropertiesTemplate.KEY_FIELD_NAME)))
-                              .patch(List.of(patchOp))
-                              .build()
-                              .getJsonPatch())
-                      .auditStamp(mclItem.getAuditStamp())
-                      .systemMetadata(mclItem.getSystemMetadata())
-                      .build(retrieverContext.getAspectRetriever().getEntityRegistry()));
-            }
+              .orElse(new DataProductAssociationArray());
+
+      DataProductProperties previousDataProductProperties =
+          mclItem.getPreviousAspect(DataProductProperties.class);
+
+      if (!ChangeType.UPSERT.equals(mclItem.getChangeType())
+          || previousDataProductProperties == null) {
+        // CREATE/CREATE_ENTITY/RESTATE
+        return generateUnsetMCPs(mclItem, newDataProductAssociationArray, retrieverContext);
+      } else {
+        // UPSERT with previous
+        DataProductAssociationArray oldDataProductAssociationArray =
+            Optional.ofNullable(previousDataProductProperties.getAssets())
+                .orElse(new DataProductAssociationArray());
+
+        DataProductAssociationArray additions =
+            newDataProductAssociationArray.stream()
+                .filter(association -> !oldDataProductAssociationArray.contains(association))
+                .collect(Collectors.toCollection(DataProductAssociationArray::new));
+
+        return generateUnsetMCPs(mclItem, additions, retrieverContext);
+      }
+    }
+    return Stream.empty();
+  }
+
+  private static Stream<MCPItem> generateUnsetMCPs(
+      @Nonnull MCLItem dataProductItem,
+      @Nonnull DataProductAssociationArray dataProductAssociations,
+      @Nonnull RetrieverContext retrieverContext) {
+    List<MCPItem> mcpItems = new ArrayList<>();
+    Map<String, List<GenericJsonPatch.PatchOp>> patchOpMap = new HashMap<>();
+
+    for (DataProductAssociation dataProductAssociation : dataProductAssociations) {
+      RelatedEntitiesScrollResult result =
+          retrieverContext
+              .getGraphRetriever()
+              .scrollRelatedEntities(
+                  null,
+                  QueryUtils.newFilter(
+                      "urn", dataProductAssociation.getDestinationUrn().toString()),
+                  null,
+                  EMPTY_FILTER,
+                  ImmutableList.of("DataProductContains"),
+                  QueryUtils.newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING),
+                  Collections.emptyList(),
+                  null,
+                  10, // Should only ever be one, if ever greater than ten will decrease over time
+                  // to become consistent
+                  null,
+                  null);
+      if (!result.getEntities().isEmpty()) {
+        for (RelatedEntities entity : result.getEntities()) {
+          if (!dataProductItem.getUrn().equals(UrnUtils.getUrn(entity.getSourceUrn()))) {
+            GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp();
+            patchOp.setOp(PatchOperationType.REMOVE.getValue());
+            patchOp.setPath(String.format("/assets/%s", entity.getDestinationUrn()));
+            patchOpMap
+                .computeIfAbsent(entity.getSourceUrn(), urn -> new ArrayList<>())
+                .add(patchOp);
           }
         }
       }
-      return mcpItems.stream();
     }
-    return Stream.empty();
+    for (String urn : patchOpMap.keySet()) {
+      EntitySpec entitySpec =
+          retrieverContext
+              .getAspectRetriever()
+              .getEntityRegistry()
+              .getEntitySpec(DATA_PRODUCT_ENTITY_NAME);
+      mcpItems.add(
+          PatchItemImpl.builder()
+              .urn(UrnUtils.getUrn(urn))
+              .entitySpec(
+                  retrieverContext
+                      .getAspectRetriever()
+                      .getEntityRegistry()
+                      .getEntitySpec(DATA_PRODUCT_ENTITY_NAME))
+              .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)
+              .aspectSpec(entitySpec.getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME))
+              .patch(
+                  GenericJsonPatch.builder()
+                      .arrayPrimaryKeys(
+                          Map.of(
+                              DataProductPropertiesTemplate.ASSETS_FIELD_NAME,
+                              List.of(DataProductPropertiesTemplate.KEY_FIELD_NAME)))
+                      .patch(patchOpMap.get(urn))
+                      .build()
+                      .getJsonPatch())
+              .auditStamp(dataProductItem.getAuditStamp())
+              .systemMetadata(dataProductItem.getSystemMetadata())
+              .build(retrieverContext.getAspectRetriever().getEntityRegistry()));
+    }
+
+    return mcpItems.stream();
   }
 }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java
index 3f0545b6f94a8..7a8c5c76c31c3 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java
@@ -43,7 +43,7 @@ EntityAspect getAspect(
 
   @Nonnull
   Map<EntityAspectIdentifier, EntityAspect> batchGet(
-      @Nonnull final Set<EntityAspectIdentifier> keys);
+      @Nonnull final Set<EntityAspectIdentifier> keys, boolean forUpdate);
 
   @Nonnull
   List<EntityAspect> getAspectsInRange(
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceAspectRetriever.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceAspectRetriever.java
index 626a1f72f5fb7..50cf8af30d606 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceAspectRetriever.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceAspectRetriever.java
@@ -5,7 +5,7 @@
 
 import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.Aspect;
-import com.linkedin.metadata.aspect.CachingAspectRetriever;
+import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.aspect.SystemAspect;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import io.datahubproject.metadata.context.OperationContext;
@@ -22,7 +22,7 @@
 
 @Getter
 @Builder
-public class EntityServiceAspectRetriever implements CachingAspectRetriever {
+public class EntityServiceAspectRetriever implements AspectRetriever {
 
   @Setter private OperationContext systemOperationContext;
   private final EntityRegistry entityRegistry;
@@ -46,7 +46,8 @@ public Map<Urn, Map<String, com.linkedin.entity.Aspect>> getLatestAspectObjects(
       String entityName = urns.stream().findFirst().map(Urn::getEntityType).get();
       try {
         return entityResponseToAspectMap(
-            entityService.getEntitiesV2(systemOperationContext, entityName, urns, aspectNames));
+            entityService.getEntitiesV2(
+                systemOperationContext, entityName, urns, aspectNames, false));
       } catch (URISyntaxException e) {
         throw new RuntimeException(e);
       }
@@ -71,7 +72,8 @@ public Map<Urn, Map<String, SystemAspect>> getLatestSystemAspects(
                 urnAspectNames.keySet(),
                 urnAspectNames.values().stream()
                     .flatMap(Collection::stream)
-                    .collect(Collectors.toSet())),
+                    .collect(Collectors.toSet()),
+                false),
             entityRegistry);
       } catch (URISyntaxException e) {
         throw new RuntimeException(e);
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java
index 34c98bba01af4..8ae09111204ca 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java
@@ -164,6 +164,8 @@ public class EntityServiceImpl implements EntityService<ChangeItemImpl> {
   private final Integer ebeanMaxTransactionRetry;
   private final boolean enableBrowseV2;
 
+  private static final long DB_TIMER_LOG_THRESHOLD_MS = 50;
+
   @Getter
   private final Map<Set<ThrottleType>, ThrottleEvent> throttleEvents = new ConcurrentHashMap<>();
 
@@ -236,7 +238,7 @@ public Map<Urn, List<RecordTemplate>> getLatestAspects(
       boolean alwaysIncludeKeyAspect) {
 
     Map<EntityAspectIdentifier, EntityAspect> batchGetResults =
-        getLatestAspect(opContext, urns, aspectNames);
+        getLatestAspect(opContext, urns, aspectNames, false);
 
     // Fetch from db and populate urn -> aspect map.
     final Map<Urn, List<RecordTemplate>> urnToAspects = new HashMap<>();
@@ -259,8 +261,7 @@ public Map<Urn, List<RecordTemplate>> getLatestAspects(
     }
 
     List<SystemAspect> systemAspects =
-        EntityUtils.toSystemAspects(
-            opContext.getRetrieverContext().get(), batchGetResults.values());
+        EntityUtils.toSystemAspects(opContext.getRetrieverContext(), batchGetResults.values());
 
     systemAspects.stream()
         // for now, don't add the key aspect here we have already added it above
@@ -283,12 +284,12 @@ public Map<Urn, List<RecordTemplate>> getLatestAspects(
   public Map<String, RecordTemplate> getLatestAspectsForUrn(
       @Nonnull OperationContext opContext,
       @Nonnull final Urn urn,
-      @Nonnull final Set<String> aspectNames) {
+      @Nonnull final Set<String> aspectNames,
+      boolean forUpdate) {
     Map<EntityAspectIdentifier, EntityAspect> batchGetResults =
-        getLatestAspect(opContext, new HashSet<>(Arrays.asList(urn)), aspectNames);
+        getLatestAspect(opContext, new HashSet<>(Arrays.asList(urn)), aspectNames, forUpdate);
 
-    return EntityUtils.toSystemAspects(
-            opContext.getRetrieverContext().get(), batchGetResults.values())
+    return EntityUtils.toSystemAspects(opContext.getRetrieverContext(), batchGetResults.values())
         .stream()
         .map(
             systemAspect -> Pair.of(systemAspect.getAspectName(), systemAspect.getRecordTemplate()))
@@ -332,7 +333,7 @@ public Pair<RecordTemplate, Long> getAspectVersionPair(
     final Optional<EntityAspect> maybeAspect = Optional.ofNullable(aspectDao.getAspect(primaryKey));
 
     return Pair.of(
-        EntityUtils.toSystemAspect(opContext.getRetrieverContext().get(), maybeAspect.orElse(null))
+        EntityUtils.toSystemAspect(opContext.getRetrieverContext(), maybeAspect.orElse(null))
             .map(SystemAspect::getRecordTemplate)
             .orElse(null),
         version);
@@ -718,7 +719,7 @@ public ListResult<RecordTemplate> listLatestAspects(
     }
 
     return new ListResult<>(
-        EntityUtils.toSystemAspects(opContext.getRetrieverContext().get(), entityAspects).stream()
+        EntityUtils.toSystemAspects(opContext.getRetrieverContext(), entityAspects).stream()
             .map(SystemAspect::getRecordTemplate)
             .collect(Collectors.toList()),
         aspectMetadataList.getMetadata(),
@@ -755,12 +756,12 @@ public List<UpdateAspectResult> ingestAspects(
                         .recordTemplate(pair.getValue())
                         .systemMetadata(systemMetadata)
                         .auditStamp(auditStamp)
-                        .build(opContext.getAspectRetrieverOpt().get()))
+                        .build(opContext.getAspectRetriever()))
             .collect(Collectors.toList());
     return ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -812,13 +813,13 @@ private void processPostCommitMCLSideEffects(
     log.debug("Considering {} MCLs post commit side effects.", mcls.size());
     List<MCLItem> batch =
         mcls.stream()
-            .map(mcl -> MCLItemImpl.builder().build(mcl, opContext.getAspectRetrieverOpt().get()))
+            .map(mcl -> MCLItemImpl.builder().build(mcl, opContext.getAspectRetriever()))
             .collect(Collectors.toList());
 
     Iterable<List<MCPItem>> iterable =
         () ->
             Iterators.partition(
-                AspectsBatch.applyPostMCPSideEffects(batch, opContext.getRetrieverContext().get())
+                AspectsBatch.applyPostMCPSideEffects(batch, opContext.getRetrieverContext())
                     .iterator(),
                 MCP_SIDE_EFFECT_KAFKA_BATCH_SIZE);
     StreamSupport.stream(iterable.spliterator(), false)
@@ -828,7 +829,7 @@ private void processPostCommitMCLSideEffects(
                   ingestProposalAsync(
                           AspectsBatchImpl.builder()
                               .items(sideEffects)
-                              .retrieverContext(opContext.getRetrieverContext().get())
+                              .retrieverContext(opContext.getRetrieverContext())
                               .build())
                       .count();
               log.info("Generated {} MCP SideEffects for async processing", count);
@@ -852,7 +853,8 @@ private List<UpdateAspectResult> ingestAspectsToLocalDB(
       boolean overwrite) {
 
     if (inputBatch.containsDuplicateAspects()) {
-      log.warn(String.format("Batch contains duplicates: %s", inputBatch));
+      log.warn("Batch contains duplicates: {}", inputBatch.duplicateAspects());
+      MetricUtils.counter(EntityServiceImpl.class, "batch_with_duplicate").inc();
     }
 
     return aspectDao
@@ -865,175 +867,186 @@ private List<UpdateAspectResult> ingestAspectsToLocalDB(
 
               // Read before write is unfortunate, however batch it
               final Map<String, Set<String>> urnAspects = batchWithDefaults.getUrnAspectsMap();
+
               // read #1
-              final Map<String, Map<String, SystemAspect>> latestAspects =
-                  EntityUtils.toSystemAspects(
-                      opContext.getRetrieverContext().get(),
-                      aspectDao.getLatestAspects(urnAspects, true));
+              // READ COMMITED is used in conjunction with SELECT FOR UPDATE (read lock) in order
+              // to ensure that the aspect's version is not modified outside the transaction.
+              // We rely on the retry mechanism if the row is modified and will re-read (require the
+              // lock)
+              Map<String, Map<String, EntityAspect>> databaseAspects =
+                  aspectDao.getLatestAspects(urnAspects, true);
+
+              final Map<String, Map<String, SystemAspect>> batchAspects =
+                  EntityUtils.toSystemAspects(opContext.getRetrieverContext(), databaseAspects);
+
               // read #2 (potentially)
               final Map<String, Map<String, Long>> nextVersions =
-                  EntityUtils.calculateNextVersions(
-                      txContext, aspectDao, latestAspects, urnAspects);
+                  EntityUtils.calculateNextVersions(txContext, aspectDao, batchAspects, urnAspects);
 
               // 1. Convert patches to full upserts
               // 2. Run any entity/aspect level hooks
               Pair<Map<String, Set<String>>, List<ChangeMCP>> updatedItems =
-                  batchWithDefaults.toUpsertBatchItems(latestAspects);
+                  batchWithDefaults.toUpsertBatchItems(batchAspects, nextVersions);
 
               // Fetch additional information if needed
-              final Map<String, Map<String, SystemAspect>> updatedLatestAspects;
-              final Map<String, Map<String, Long>> updatedNextVersions;
+              final List<ChangeMCP> changeMCPs;
+
               if (!updatedItems.getFirst().isEmpty()) {
+                // These items are new items from side effects
+                Map<String, Set<String>> sideEffects = updatedItems.getFirst();
+
+                final Map<String, Map<String, SystemAspect>> updatedLatestAspects;
+                final Map<String, Map<String, Long>> updatedNextVersions;
+
                 Map<String, Map<String, SystemAspect>> newLatestAspects =
                     EntityUtils.toSystemAspects(
-                        opContext.getRetrieverContext().get(),
+                        opContext.getRetrieverContext(),
                         aspectDao.getLatestAspects(updatedItems.getFirst(), true));
                 // merge
-                updatedLatestAspects = AspectsBatch.merge(latestAspects, newLatestAspects);
+                updatedLatestAspects = AspectsBatch.merge(batchAspects, newLatestAspects);
 
                 Map<String, Map<String, Long>> newNextVersions =
                     EntityUtils.calculateNextVersions(
                         txContext, aspectDao, updatedLatestAspects, updatedItems.getFirst());
                 // merge
                 updatedNextVersions = AspectsBatch.merge(nextVersions, newNextVersions);
+
+                changeMCPs =
+                    updatedItems.getSecond().stream()
+                        .peek(
+                            changeMCP -> {
+                              // Add previous version to each side-effect
+                              if (sideEffects
+                                  .getOrDefault(
+                                      changeMCP.getUrn().toString(), Collections.emptySet())
+                                  .contains(changeMCP.getAspectName())) {
+
+                                AspectsBatch.incrementBatchVersion(
+                                    changeMCP, updatedLatestAspects, updatedNextVersions);
+                              }
+                            })
+                        .collect(Collectors.toList());
               } else {
-                updatedLatestAspects = latestAspects;
-                updatedNextVersions = nextVersions;
+                changeMCPs = updatedItems.getSecond();
               }
 
-              // Add previous version to each upsert
-              List<ChangeMCP> changeMCPs =
-                  updatedItems.getSecond().stream()
-                      .peek(
-                          changeMCP -> {
-                            String urnStr = changeMCP.getUrn().toString();
-                            long nextVersion =
-                                updatedNextVersions
-                                    .getOrDefault(urnStr, Map.of())
-                                    .getOrDefault(changeMCP.getAspectName(), 0L);
-
-                            changeMCP.setPreviousSystemAspect(
-                                updatedLatestAspects
-                                    .getOrDefault(urnStr, Map.of())
-                                    .getOrDefault(changeMCP.getAspectName(), null));
-
-                            changeMCP.setNextAspectVersion(nextVersion);
-
-                            // support inner-batch upserts
-                            updatedLatestAspects
-                                .computeIfAbsent(urnStr, key -> new HashMap<>())
-                                .put(
-                                    changeMCP.getAspectName(),
-                                    changeMCP.getSystemAspect(nextVersion));
-                            updatedNextVersions
-                                .computeIfAbsent(urnStr, key -> new HashMap<>())
-                                .put(changeMCP.getAspectName(), nextVersion + 1);
-                          })
-                      .collect(Collectors.toList());
-
               // No changes, return
               if (changeMCPs.isEmpty()) {
+                MetricUtils.counter(EntityServiceImpl.class, "batch_empty").inc();
                 return Collections.<UpdateAspectResult>emptyList();
               }
 
               // do final pre-commit checks with previous aspect value
               ValidationExceptionCollection exceptions =
-                  AspectsBatch.validatePreCommit(changeMCPs, opContext.getRetrieverContext().get());
-              if (!exceptions.isEmpty()) {
-                throw new ValidationException(collectMetrics(exceptions).toString());
+                  AspectsBatch.validatePreCommit(changeMCPs, opContext.getRetrieverContext());
+
+              if (exceptions.hasFatalExceptions()) {
+                // IF this is a client request/API request we fail the `transaction batch`
+                if (opContext.getRequestContext() != null) {
+                  MetricUtils.counter(EntityServiceImpl.class, "batch_request_validation_exception")
+                      .inc();
+                  throw new ValidationException(collectMetrics(exceptions).toString());
+                }
+
+                MetricUtils.counter(EntityServiceImpl.class, "batch_consumer_validation_exception")
+                    .inc();
+                log.error("mce-consumer batch exceptions: {}", collectMetrics(exceptions));
               }
 
-              // Database Upsert results
+              // Database Upsert successfully validated results
               log.info(
                   "Ingesting aspects batch to database: {}",
                   AspectsBatch.toAbbreviatedString(changeMCPs, 2048));
               Timer.Context ingestToLocalDBTimer =
                   MetricUtils.timer(this.getClass(), "ingestAspectsToLocalDB").time();
               List<UpdateAspectResult> upsertResults =
-                  changeMCPs.stream()
+                  exceptions
+                      .streamSuccessful(changeMCPs.stream())
                       .map(
-                          item -> {
-                            final EntityAspect.EntitySystemAspect latest =
-                                (EntityAspect.EntitySystemAspect) item.getPreviousSystemAspect();
-
-                            final UpdateAspectResult result;
-                            if (overwrite || latest == null) {
-                              result =
-                                  ingestAspectToLocalDB(
-                                          txContext,
-                                          item.getUrn(),
-                                          item.getAspectName(),
-                                          item.getRecordTemplate(),
-                                          item.getAuditStamp(),
-                                          item.getSystemMetadata(),
-                                          latest == null ? null : latest,
-                                          item.getNextAspectVersion())
-                                      .toBuilder()
-                                      .request(item)
-                                      .build();
-
-                            } else {
-                              RecordTemplate oldValue = latest.getRecordTemplate();
-                              SystemMetadata oldMetadata = latest.getSystemMetadata();
-                              result =
-                                  UpdateAspectResult.<ChangeItemImpl>builder()
-                                      .urn(item.getUrn())
-                                      .request(item)
-                                      .oldValue(oldValue)
-                                      .newValue(oldValue)
-                                      .oldSystemMetadata(oldMetadata)
-                                      .newSystemMetadata(oldMetadata)
-                                      .operation(MetadataAuditOperation.UPDATE)
-                                      .auditStamp(item.getAuditStamp())
-                                      .maxVersion(latest.getVersion())
-                                      .build();
+                          writeItem -> {
+
+                            /*
+                              database*Aspect - should be used for comparisons of before batch operation information
+                            */
+                            final EntityAspect databaseAspect =
+                                databaseAspects
+                                    .getOrDefault(writeItem.getUrn().toString(), Map.of())
+                                    .get(writeItem.getAspectName());
+                            final EntityAspect.EntitySystemAspect databaseSystemAspect =
+                                databaseAspect == null
+                                    ? null
+                                    : EntityAspect.EntitySystemAspect.builder()
+                                        .build(
+                                            writeItem.getEntitySpec(),
+                                            writeItem.getAspectSpec(),
+                                            databaseAspect);
+
+                            /*
+                              This condition is specifically for an older conditional write ingestAspectIfNotPresent()
+                              overwrite is always true otherwise
+                            */
+                            if (overwrite || databaseAspect == null) {
+                              return Optional.ofNullable(
+                                      ingestAspectToLocalDB(
+                                          txContext, writeItem, databaseSystemAspect))
+                                  .map(
+                                      optResult -> optResult.toBuilder().request(writeItem).build())
+                                  .orElse(null);
                             }
 
-                            return result;
+                            return null;
                           })
+                      .filter(Objects::nonNull)
                       .collect(Collectors.toList());
 
-              // commit upserts prior to retention or kafka send, if supported by impl
-              if (txContext != null) {
-                txContext.commitAndContinue();
-              }
-              long took = ingestToLocalDBTimer.stop();
-              log.info(
-                  "Ingestion of aspects batch to database took {} ms",
-                  TimeUnit.NANOSECONDS.toMillis(took));
-
-              // Retention optimization and tx
-              if (retentionService != null) {
-                List<RetentionService.RetentionContext> retentionBatch =
-                    upsertResults.stream()
-                        // Only consider retention when there was a previous version
-                        .filter(
-                            result ->
-                                latestAspects.containsKey(result.getUrn().toString())
-                                    && latestAspects
-                                        .get(result.getUrn().toString())
-                                        .containsKey(result.getRequest().getAspectName()))
-                        .filter(
-                            result -> {
-                              RecordTemplate oldAspect = result.getOldValue();
-                              RecordTemplate newAspect = result.getNewValue();
-                              // Apply retention policies if there was an update to existing aspect
-                              // value
-                              return oldAspect != newAspect
-                                  && oldAspect != null
-                                  && retentionService != null;
-                            })
-                        .map(
-                            result ->
-                                RetentionService.RetentionContext.builder()
-                                    .urn(result.getUrn())
-                                    .aspectName(result.getRequest().getAspectName())
-                                    .maxVersion(Optional.of(result.getMaxVersion()))
-                                    .build())
-                        .collect(Collectors.toList());
-                retentionService.applyRetentionWithPolicyDefaults(opContext, retentionBatch);
+              if (!upsertResults.isEmpty()) {
+                // commit upserts prior to retention or kafka send, if supported by impl
+                if (txContext != null) {
+                  txContext.commitAndContinue();
+                }
+                long took = TimeUnit.NANOSECONDS.toMillis(ingestToLocalDBTimer.stop());
+                if (took > DB_TIMER_LOG_THRESHOLD_MS) {
+                  log.info("Ingestion of aspects batch to database took {} ms", took);
+                }
+
+                // Retention optimization and tx
+                if (retentionService != null) {
+                  List<RetentionService.RetentionContext> retentionBatch =
+                      upsertResults.stream()
+                          // Only consider retention when there was a previous version
+                          .filter(
+                              result ->
+                                  batchAspects.containsKey(result.getUrn().toString())
+                                      && batchAspects
+                                          .get(result.getUrn().toString())
+                                          .containsKey(result.getRequest().getAspectName()))
+                          .filter(
+                              result -> {
+                                RecordTemplate oldAspect = result.getOldValue();
+                                RecordTemplate newAspect = result.getNewValue();
+                                // Apply retention policies if there was an update to existing
+                                // aspect
+                                // value
+                                return oldAspect != newAspect
+                                    && oldAspect != null
+                                    && retentionService != null;
+                              })
+                          .map(
+                              result ->
+                                  RetentionService.RetentionContext.builder()
+                                      .urn(result.getUrn())
+                                      .aspectName(result.getRequest().getAspectName())
+                                      .maxVersion(Optional.of(result.getMaxVersion()))
+                                      .build())
+                          .collect(Collectors.toList());
+                  retentionService.applyRetentionWithPolicyDefaults(opContext, retentionBatch);
+                } else {
+                  log.warn("Retention service is missing!");
+                }
               } else {
-                log.warn("Retention service is missing!");
+                MetricUtils.counter(EntityServiceImpl.class, "batch_empty_transaction").inc();
+                // This includes no-op batches. i.e. patch removing non-existent items
+                log.debug("Empty transaction detected");
               }
 
               return upsertResults;
@@ -1100,9 +1113,11 @@ private List<UpdateAspectResult> emitMCL(
    * @param auditStamp an {@link AuditStamp} containing metadata about the writer & current time
    * @param systemMetadata
    * @return the {@link RecordTemplate} representation of the written aspect object
+   * @deprecated See Conditional Write ChangeType CREATE
    */
   @Nullable
   @Override
+  @Deprecated
   public RecordTemplate ingestAspectIfNotPresent(
       @Nonnull OperationContext opContext,
       @Nonnull Urn urn,
@@ -1125,12 +1140,12 @@ public RecordTemplate ingestAspectIfNotPresent(
                     .recordTemplate(newValue)
                     .systemMetadata(systemMetadata)
                     .auditStamp(auditStamp)
-                    .build(opContext.getAspectRetrieverOpt().get()),
-                opContext.getRetrieverContext().get())
+                    .build(opContext.getAspectRetriever()),
+                opContext.getRetrieverContext())
             .build();
     List<UpdateAspectResult> ingested = ingestAspects(opContext, aspectsBatch, true, false);
 
-    return ingested.stream().findFirst().get().getNewValue();
+    return ingested.stream().findFirst().map(UpdateAspectResult::getNewValue).orElse(null);
   }
 
   /**
@@ -1151,7 +1166,7 @@ public IngestResult ingestProposal(
     return ingestProposal(
             opContext,
             AspectsBatchImpl.builder()
-                .mcps(List.of(proposal), auditStamp, opContext.getRetrieverContext().get())
+                .mcps(List.of(proposal), auditStamp, opContext.getRetrieverContext())
                 .build(),
             async)
         .stream()
@@ -1173,15 +1188,15 @@ public IngestResult ingestProposal(
    * @return an {@link IngestResult} containing the results
    */
   @Override
-  public Set<IngestResult> ingestProposal(
+  public List<IngestResult> ingestProposal(
       @Nonnull OperationContext opContext, AspectsBatch aspectsBatch, final boolean async) {
     Stream<IngestResult> timeseriesIngestResults =
         ingestTimeseriesProposal(opContext, aspectsBatch, async);
     Stream<IngestResult> nonTimeseriesIngestResults =
         async ? ingestProposalAsync(aspectsBatch) : ingestProposalSync(opContext, aspectsBatch);
 
-    return Stream.concat(timeseriesIngestResults, nonTimeseriesIngestResults)
-        .collect(Collectors.toSet());
+    return Stream.concat(nonTimeseriesIngestResults, timeseriesIngestResults)
+        .collect(Collectors.toList());
   }
 
   /**
@@ -1192,11 +1207,13 @@ public Set<IngestResult> ingestProposal(
    */
   private Stream<IngestResult> ingestTimeseriesProposal(
       @Nonnull OperationContext opContext, AspectsBatch aspectsBatch, final boolean async) {
+
     List<? extends BatchItem> unsupported =
         aspectsBatch.getItems().stream()
             .filter(
                 item ->
-                    item.getAspectSpec().isTimeseries()
+                    item.getAspectSpec() != null
+                        && item.getAspectSpec().isTimeseries()
                         && item.getChangeType() != ChangeType.UPSERT)
             .collect(Collectors.toList());
     if (!unsupported.isEmpty()) {
@@ -1212,7 +1229,7 @@ private Stream<IngestResult> ingestTimeseriesProposal(
       // Create default non-timeseries aspects for timeseries aspects
       List<MCPItem> timeseriesKeyAspects =
           aspectsBatch.getMCPItems().stream()
-              .filter(item -> item.getAspectSpec().isTimeseries())
+              .filter(item -> item.getAspectSpec() != null && item.getAspectSpec().isTimeseries())
               .map(
                   item ->
                       ChangeItemImpl.builder()
@@ -1226,7 +1243,7 @@ private Stream<IngestResult> ingestTimeseriesProposal(
                           .recordTemplate(
                               EntityApiUtils.buildKeyAspect(
                                   opContext.getEntityRegistry(), item.getUrn()))
-                          .build(opContext.getAspectRetrieverOpt().get()))
+                          .build(opContext.getAspectRetriever()))
               .collect(Collectors.toList());
 
       ingestProposalSync(
@@ -1238,10 +1255,10 @@ private Stream<IngestResult> ingestTimeseriesProposal(
     }
 
     // Emit timeseries MCLs
-    List<Pair<ChangeItemImpl, Optional<Pair<Future<?>, Boolean>>>> timeseriesResults =
+    List<Pair<MCPItem, Optional<Pair<Future<?>, Boolean>>>> timeseriesResults =
         aspectsBatch.getItems().stream()
-            .filter(item -> item.getAspectSpec().isTimeseries())
-            .map(item -> (ChangeItemImpl) item)
+            .filter(item -> item.getAspectSpec() != null && item.getAspectSpec().isTimeseries())
+            .map(item -> (MCPItem) item)
             .map(
                 item ->
                     Pair.of(
@@ -1261,6 +1278,7 @@ private Stream<IngestResult> ingestTimeseriesProposal(
     return timeseriesResults.stream()
         .map(
             result -> {
+              MCPItem item = result.getFirst();
               Optional<Pair<Future<?>, Boolean>> emissionStatus = result.getSecond();
 
               emissionStatus.ifPresent(
@@ -1272,10 +1290,16 @@ private Stream<IngestResult> ingestTimeseriesProposal(
                     }
                   });
 
-              ChangeItemImpl request = result.getFirst();
               return IngestResult.builder()
-                  .urn(request.getUrn())
-                  .request(request)
+                  .urn(item.getUrn())
+                  .request(item)
+                  .result(
+                      UpdateAspectResult.builder()
+                          .urn(item.getUrn())
+                          .newValue(item.getRecordTemplate())
+                          .auditStamp(item.getAuditStamp())
+                          .newSystemMetadata(item.getSystemMetadata())
+                          .build())
                   .publishedMCL(
                       emissionStatus.map(status -> status.getFirst() != null).orElse(false))
                   .processedMCL(emissionStatus.map(Pair::getSecond).orElse(false))
@@ -1292,7 +1316,7 @@ private Stream<IngestResult> ingestTimeseriesProposal(
   private Stream<IngestResult> ingestProposalAsync(AspectsBatch aspectsBatch) {
     List<? extends MCPItem> nonTimeseries =
         aspectsBatch.getMCPItems().stream()
-            .filter(item -> !item.getAspectSpec().isTimeseries())
+            .filter(item -> item.getAspectSpec() == null || !item.getAspectSpec().isTimeseries())
             .collect(Collectors.toList());
 
     List<Future<?>> futures =
@@ -1328,6 +1352,7 @@ private Stream<IngestResult> ingestProposalAsync(AspectsBatch aspectsBatch) {
 
   private Stream<IngestResult> ingestProposalSync(
       @Nonnull OperationContext opContext, AspectsBatch aspectsBatch) {
+
     AspectsBatchImpl nonTimeseries =
         AspectsBatchImpl.builder()
             .retrieverContext(aspectsBatch.getRetrieverContext())
@@ -1357,6 +1382,7 @@ private Stream<IngestResult> ingestProposalSync(
               return IngestResult.builder()
                   .urn(item.getUrn())
                   .request(item)
+                  .result(result)
                   .publishedMCL(result.getMclFuture() != null)
                   .sqlCommitted(true)
                   .isUpdate(result.getOldValue() != null)
@@ -1440,7 +1466,7 @@ public List<RestoreIndicesResult> restoreIndices(
 
                 List<SystemAspect> systemAspects =
                     EntityUtils.toSystemAspectFromEbeanAspects(
-                        opContext.getRetrieverContext().get(), batch.collect(Collectors.toList()));
+                        opContext.getRetrieverContext(), batch.collect(Collectors.toList()));
 
                 RestoreIndicesResult result = restoreIndices(opContext, systemAspects, logger);
                 result.timeSqlQueryMs = timeSqlQueryMs;
@@ -1484,8 +1510,8 @@ public List<RestoreIndicesResult> restoreIndices(
         long startTime = System.currentTimeMillis();
         List<SystemAspect> systemAspects =
             EntityUtils.toSystemAspects(
-                opContext.getRetrieverContext().get(),
-                getLatestAspect(opContext, entityBatch.getValue(), aspectNames).values());
+                opContext.getRetrieverContext(),
+                getLatestAspect(opContext, entityBatch.getValue(), aspectNames, false).values());
         long timeSqlQueryMs = System.currentTimeMillis() - startTime;
 
         RestoreIndicesResult result = restoreIndices(opContext, systemAspects, s -> {});
@@ -1620,12 +1646,12 @@ private RestoreIndicesResult restoreIndices(
                   .auditStamp(auditStamp)
                   .systemMetadata(latestSystemMetadata)
                   .recordTemplate(EntityApiUtils.buildKeyAspect(opContext.getEntityRegistry(), urn))
-                  .build(opContext.getAspectRetrieverOpt().get()));
+                  .build(opContext.getAspectRetriever()));
       Stream<IngestResult> defaultAspectsResult =
           ingestProposalSync(
               opContext,
               AspectsBatchImpl.builder()
-                  .retrieverContext(opContext.getRetrieverContext().get())
+                  .retrieverContext(opContext.getRetrieverContext())
                   .items(keyAspect)
                   .build());
       defaultAspectsCreated += defaultAspectsResult.count();
@@ -1937,7 +1963,7 @@ private void ingestSnapshotUnion(
 
     AspectsBatchImpl aspectsBatch =
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(
                 aspectRecordsToIngest.stream()
                     .map(
@@ -1948,7 +1974,7 @@ private void ingestSnapshotUnion(
                                 .recordTemplate(pair.getValue())
                                 .auditStamp(auditStamp)
                                 .systemMetadata(systemMetadata)
-                                .build(opContext.getAspectRetrieverOpt().get()))
+                                .build(opContext.getAspectRetriever()))
                     .collect(Collectors.toList()))
             .build();
 
@@ -2099,7 +2125,7 @@ public RollbackRunResult deleteUrn(@Nonnull OperationContext opContext, Urn urn)
     }
 
     SystemMetadata latestKeySystemMetadata =
-        EntityUtils.toSystemAspect(opContext.getRetrieverContext().get(), latestKey)
+        EntityUtils.toSystemAspect(opContext.getRetrieverContext(), latestKey)
             .map(SystemAspect::getSystemMetadata)
             .get();
     RollbackResult result =
@@ -2155,7 +2181,8 @@ public Set<Urn> exists(
       @Nonnull OperationContext opContext,
       @Nonnull final Collection<Urn> urns,
       @Nullable String aspectName,
-      boolean includeSoftDeleted) {
+      boolean includeSoftDeleted,
+      boolean forUpdate) {
     final Set<EntityAspectIdentifier> dbKeys =
         urns.stream()
             .map(
@@ -2171,11 +2198,11 @@ public Set<Urn> exists(
                             : aspectName,
                         ASPECT_LATEST_VERSION))
             .collect(Collectors.toSet());
-    final Map<EntityAspectIdentifier, EntityAspect> aspects = aspectDao.batchGet(dbKeys);
+    final Map<EntityAspectIdentifier, EntityAspect> aspects = aspectDao.batchGet(dbKeys, forUpdate);
     final Set<String> existingUrnStrings =
         aspects.values().stream()
-            .filter(aspect -> aspect != null)
-            .map(aspect -> aspect.getUrn())
+            .filter(Objects::nonNull)
+            .map(EntityAspect::getUrn)
             .collect(Collectors.toSet());
 
     Set<Urn> existing =
@@ -2223,11 +2250,11 @@ private RollbackResult deleteAspectWithoutMCL(
             .urn(entityUrn)
             .aspectName(aspectName)
             .auditStamp(auditStamp)
-            .build(opContext.getAspectRetrieverOpt().get());
+            .build(opContext.getAspectRetriever());
 
     // Delete validation hooks
     ValidationExceptionCollection exceptions =
-        AspectsBatch.validateProposed(List.of(deleteItem), opContext.getRetrieverContext().get());
+        AspectsBatch.validateProposed(List.of(deleteItem), opContext.getRetrieverContext());
     if (!exceptions.isEmpty()) {
       throw new ValidationException(collectMetrics(exceptions).toString());
     }
@@ -2241,7 +2268,7 @@ private RollbackResult deleteAspectWithoutMCL(
               final EntityAspect.EntitySystemAspect latest =
                   (EntityAspect.EntitySystemAspect)
                       EntityUtils.toSystemAspect(
-                              opContext.getRetrieverContext().get(),
+                              opContext.getRetrieverContext(),
                               aspectDao.getLatestAspect(urn, aspectName, false))
                           .orElse(null);
 
@@ -2269,7 +2296,7 @@ private RollbackResult deleteAspectWithoutMCL(
                 EntityAspect.EntitySystemAspect candidateAspect =
                     (EntityAspect.EntitySystemAspect)
                         EntityUtils.toSystemAspect(
-                                opContext.getRetrieverContext().get(),
+                                opContext.getRetrieverContext(),
                                 aspectDao.getAspect(urn, aspectName, maxVersion))
                             .orElse(null);
                 SystemMetadata previousSysMetadata =
@@ -2295,13 +2322,9 @@ private RollbackResult deleteAspectWithoutMCL(
                                       .urn(UrnUtils.getUrn(toDelete.getUrn()))
                                       .aspectName(toDelete.getAspect())
                                       .auditStamp(auditStamp)
-                                      .build(
-                                          opContext
-                                              .getRetrieverContext()
-                                              .get()
-                                              .getAspectRetriever()))
+                                      .build(opContext.getAspectRetriever()))
                           .collect(Collectors.toList()),
-                      opContext.getRetrieverContext().get());
+                      opContext.getRetrieverContext());
               if (!preCommitExceptions.isEmpty()) {
                 throw new ValidationException(collectMetrics(preCommitExceptions).toString());
               }
@@ -2431,7 +2454,8 @@ protected AuditStamp createSystemAuditStamp() {
   private Map<EntityAspectIdentifier, EntityAspect> getLatestAspect(
       @Nonnull OperationContext opContext,
       @Nonnull final Set<Urn> urns,
-      @Nonnull final Set<String> aspectNames) {
+      @Nonnull final Set<String> aspectNames,
+      boolean forUpdate) {
 
     log.debug("Invoked getLatestAspects with urns: {}, aspectNames: {}", urns, aspectNames);
 
@@ -2455,7 +2479,8 @@ private Map<EntityAspectIdentifier, EntityAspect> getLatestAspect(
     Map<EntityAspectIdentifier, EntityAspect> batchGetResults = new HashMap<>();
     Iterators.partition(dbKeys.iterator(), MAX_KEYS_PER_QUERY)
         .forEachRemaining(
-            batch -> batchGetResults.putAll(aspectDao.batchGet(ImmutableSet.copyOf(batch))));
+            batch ->
+                batchGetResults.putAll(aspectDao.batchGet(ImmutableSet.copyOf(batch), forUpdate)));
     return batchGetResults;
   }
 
@@ -2474,10 +2499,10 @@ private long calculateVersionNumber(
 
   private Map<EntityAspectIdentifier, EnvelopedAspect> getEnvelopedAspects(
       @Nonnull OperationContext opContext, final Set<EntityAspectIdentifier> dbKeys) {
-    final Map<EntityAspectIdentifier, EntityAspect> dbEntries = aspectDao.batchGet(dbKeys);
+    final Map<EntityAspectIdentifier, EntityAspect> dbEntries = aspectDao.batchGet(dbKeys, false);
 
     List<SystemAspect> envelopedAspects =
-        EntityUtils.toSystemAspects(opContext.getRetrieverContext().get(), dbEntries.values());
+        EntityUtils.toSystemAspects(opContext.getRetrieverContext(), dbEntries.values());
 
     return envelopedAspects.stream()
         .collect(
@@ -2489,89 +2514,170 @@ private Map<EntityAspectIdentifier, EnvelopedAspect> getEnvelopedAspects(
                     ((EntityAspect.EntitySystemAspect) systemAspect).toEnvelopedAspects()));
   }
 
-  @Nonnull
+  /**
+   * @param txContext Transaction context, keeps track of retries, exceptions etc.
+   * @param writeItem The aspect being written
+   * @param databaseAspect The aspect as it exists in the database.
+   * @return result object
+   */
+  @Nullable
   private UpdateAspectResult ingestAspectToLocalDB(
       @Nullable TransactionContext txContext,
-      @Nonnull final Urn urn,
-      @Nonnull final String aspectName,
-      @Nonnull final RecordTemplate newValue,
-      @Nonnull final AuditStamp auditStamp,
-      @Nonnull final SystemMetadata providedSystemMetadata,
-      @Nullable final EntityAspect.EntitySystemAspect latest,
-      @Nonnull final Long nextVersion) {
+      @Nonnull final ChangeMCP writeItem,
+      @Nullable final EntityAspect.EntitySystemAspect databaseAspect) {
+
+    if (writeItem.getRecordTemplate() == null) {
+      log.error(
+          "Unexpected write of null aspect with name {}, urn {}",
+          writeItem.getAspectName(),
+          writeItem.getUrn());
+      return null;
+    }
 
     // Set the "last run id" to be the run id provided with the new system metadata. This will be
     // stored in index
     // for all aspects that have a run id, regardless of whether they change.
-    providedSystemMetadata.setLastRunId(
-        providedSystemMetadata.getRunId(GetMode.NULL), SetMode.IGNORE_NULL);
+    writeItem
+        .getSystemMetadata()
+        .setLastRunId(writeItem.getSystemMetadata().getRunId(GetMode.NULL), SetMode.IGNORE_NULL);
 
     // 2. Compare the latest existing and new.
-    final RecordTemplate oldValue = latest == null ? null : latest.getRecordTemplate();
+    final EntityAspect.EntitySystemAspect previousBatchAspect =
+        (EntityAspect.EntitySystemAspect) writeItem.getPreviousSystemAspect();
+    final RecordTemplate previousValue =
+        previousBatchAspect == null ? null : previousBatchAspect.getRecordTemplate();
 
     // 3. If there is no difference between existing and new, we just update
     // the lastObserved in system metadata. RunId should stay as the original runId
-    if (oldValue != null && DataTemplateUtil.areEqual(oldValue, newValue)) {
-      SystemMetadata latestSystemMetadata = latest.getSystemMetadata();
-      latestSystemMetadata.setLastObserved(providedSystemMetadata.getLastObserved());
-      latestSystemMetadata.setLastRunId(
-          providedSystemMetadata.getLastRunId(GetMode.NULL), SetMode.IGNORE_NULL);
+    if (previousValue != null
+        && DataTemplateUtil.areEqual(previousValue, writeItem.getRecordTemplate())) {
+
+      Optional<SystemMetadata> latestSystemMetadataDiff =
+          systemMetadataDiff(
+              txContext,
+              writeItem.getUrn(),
+              previousBatchAspect.getSystemMetadata(),
+              writeItem.getSystemMetadata(),
+              databaseAspect == null ? null : databaseAspect.getSystemMetadata());
+
+      if (latestSystemMetadataDiff.isPresent()) {
+        // Inserts & update order is not guaranteed, flush the insert for potential updates within
+        // same tx
+        if (databaseAspect == null && txContext != null) {
+          conditionalLogLevel(
+              txContext,
+              String.format(
+                  "Flushing for systemMetadata update aspect with name %s, urn %s",
+                  writeItem.getAspectName(), writeItem.getUrn()));
+          txContext.flush();
+        }
+
+        conditionalLogLevel(
+            txContext,
+            String.format(
+                "Update aspect with name %s, urn %s, txContext: %s, databaseAspect: %s, newMetadata: %s newSystemMetadata: %s",
+                previousBatchAspect.getAspectName(),
+                previousBatchAspect.getUrn(),
+                txContext != null,
+                databaseAspect == null ? null : databaseAspect.getEntityAspect(),
+                previousBatchAspect.getEntityAspect().getMetadata(),
+                latestSystemMetadataDiff.get()));
+
+        aspectDao.saveAspect(
+            txContext,
+            previousBatchAspect.getUrnRaw(),
+            previousBatchAspect.getAspectName(),
+            previousBatchAspect.getMetadataRaw(),
+            previousBatchAspect.getCreatedBy(),
+            null,
+            previousBatchAspect.getCreatedOn(),
+            RecordUtils.toJsonString(latestSystemMetadataDiff.get()),
+            previousBatchAspect.getVersion(),
+            false);
+
+        // metrics
+        aspectDao.incrementWriteMetrics(
+            previousBatchAspect.getAspectName(),
+            1,
+            previousBatchAspect.getMetadataRaw().getBytes(StandardCharsets.UTF_8).length);
+
+        return UpdateAspectResult.builder()
+            .urn(writeItem.getUrn())
+            .oldValue(previousValue)
+            .newValue(previousValue)
+            .oldSystemMetadata(previousBatchAspect.getSystemMetadata())
+            .newSystemMetadata(latestSystemMetadataDiff.get())
+            .operation(MetadataAuditOperation.UPDATE)
+            .auditStamp(writeItem.getAuditStamp())
+            .maxVersion(0)
+            .build();
+      } else {
+        MetricUtils.counter(EntityServiceImpl.class, "batch_with_noop_sysmetadata").inc();
+        return null;
+      }
+    }
 
-      latest.getEntityAspect().setSystemMetadata(RecordUtils.toJsonString(latestSystemMetadata));
+    // 4. Save the newValue as the latest version
+    if (writeItem.getRecordTemplate() != null
+        && !DataTemplateUtil.areEqual(previousValue, writeItem.getRecordTemplate())) {
+      conditionalLogLevel(
+          txContext,
+          String.format(
+              "Insert aspect with name %s, urn %s", writeItem.getAspectName(), writeItem.getUrn()));
+
+      // Inserts & update order is not guaranteed, flush the insert for potential updates within
+      // same tx
+      if (databaseAspect == null
+          && !ASPECT_LATEST_VERSION.equals(writeItem.getNextAspectVersion())
+          && txContext != null) {
+        conditionalLogLevel(
+            txContext,
+            String.format(
+                "Flushing for update aspect with name %s, urn %s",
+                writeItem.getAspectName(), writeItem.getUrn()));
+        txContext.flush();
+      }
 
-      log.info("Ingesting aspect with name {}, urn {}", aspectName, urn);
-      aspectDao.saveAspect(txContext, latest.getEntityAspect(), false);
+      String newValueStr = EntityApiUtils.toJsonAspect(writeItem.getRecordTemplate());
+      long versionOfOld =
+          aspectDao.saveLatestAspect(
+              txContext,
+              writeItem.getUrn().toString(),
+              writeItem.getAspectName(),
+              previousBatchAspect == null ? null : EntityApiUtils.toJsonAspect(previousValue),
+              previousBatchAspect == null ? null : previousBatchAspect.getCreatedBy(),
+              previousBatchAspect == null
+                  ? null
+                  : previousBatchAspect.getEntityAspect().getCreatedFor(),
+              previousBatchAspect == null ? null : previousBatchAspect.getCreatedOn(),
+              previousBatchAspect == null ? null : previousBatchAspect.getSystemMetadataRaw(),
+              newValueStr,
+              writeItem.getAuditStamp().getActor().toString(),
+              writeItem.getAuditStamp().hasImpersonator()
+                  ? writeItem.getAuditStamp().getImpersonator().toString()
+                  : null,
+              new Timestamp(writeItem.getAuditStamp().getTime()),
+              EntityApiUtils.toJsonAspect(writeItem.getSystemMetadata()),
+              writeItem.getNextAspectVersion());
 
       // metrics
       aspectDao.incrementWriteMetrics(
-          aspectName, 1, latest.getMetadataRaw().getBytes(StandardCharsets.UTF_8).length);
+          writeItem.getAspectName(), 1, newValueStr.getBytes(StandardCharsets.UTF_8).length);
 
       return UpdateAspectResult.builder()
-          .urn(urn)
-          .oldValue(oldValue)
-          .newValue(oldValue)
-          .oldSystemMetadata(latest.getSystemMetadata())
-          .newSystemMetadata(latestSystemMetadata)
+          .urn(writeItem.getUrn())
+          .oldValue(previousValue)
+          .newValue(writeItem.getRecordTemplate())
+          .oldSystemMetadata(
+              previousBatchAspect == null ? null : previousBatchAspect.getSystemMetadata())
+          .newSystemMetadata(writeItem.getSystemMetadata())
           .operation(MetadataAuditOperation.UPDATE)
-          .auditStamp(auditStamp)
-          .maxVersion(0)
+          .auditStamp(writeItem.getAuditStamp())
+          .maxVersion(versionOfOld)
           .build();
     }
 
-    // 4. Save the newValue as the latest version
-    log.debug("Ingesting aspect with name {}, urn {}", aspectName, urn);
-    String newValueStr = EntityApiUtils.toJsonAspect(newValue);
-    long versionOfOld =
-        aspectDao.saveLatestAspect(
-            txContext,
-            urn.toString(),
-            aspectName,
-            latest == null ? null : EntityApiUtils.toJsonAspect(oldValue),
-            latest == null ? null : latest.getCreatedBy(),
-            latest == null ? null : latest.getEntityAspect().getCreatedFor(),
-            latest == null ? null : latest.getCreatedOn(),
-            latest == null ? null : latest.getSystemMetadataRaw(),
-            newValueStr,
-            auditStamp.getActor().toString(),
-            auditStamp.hasImpersonator() ? auditStamp.getImpersonator().toString() : null,
-            new Timestamp(auditStamp.getTime()),
-            EntityApiUtils.toJsonAspect(providedSystemMetadata),
-            nextVersion);
-
-    // metrics
-    aspectDao.incrementWriteMetrics(
-        aspectName, 1, newValueStr.getBytes(StandardCharsets.UTF_8).length);
-
-    return UpdateAspectResult.builder()
-        .urn(urn)
-        .oldValue(oldValue)
-        .newValue(newValue)
-        .oldSystemMetadata(latest == null ? null : latest.getSystemMetadata())
-        .newSystemMetadata(providedSystemMetadata)
-        .operation(MetadataAuditOperation.UPDATE)
-        .auditStamp(auditStamp)
-        .maxVersion(versionOfOld)
-        .build();
+    return null;
   }
 
   private static boolean shouldAspectEmitChangeLog(@Nonnull final AspectSpec aspectSpec) {
@@ -2579,4 +2685,43 @@ private static boolean shouldAspectEmitChangeLog(@Nonnull final AspectSpec aspec
         aspectSpec.getRelationshipFieldSpecs();
     return relationshipFieldSpecs.stream().anyMatch(RelationshipFieldSpec::isLineageRelationship);
   }
+
+  private static Optional<SystemMetadata> systemMetadataDiff(
+      @Nullable TransactionContext txContext,
+      @Nonnull Urn urn,
+      @Nullable SystemMetadata previous,
+      @Nonnull SystemMetadata current,
+      @Nullable SystemMetadata database) {
+
+    SystemMetadata latestSystemMetadata = GenericRecordUtils.copy(previous, SystemMetadata.class);
+
+    latestSystemMetadata.setLastRunId(latestSystemMetadata.getRunId(), SetMode.REMOVE_IF_NULL);
+    latestSystemMetadata.setLastObserved(current.getLastObserved(), SetMode.IGNORE_NULL);
+    latestSystemMetadata.setRunId(current.getRunId(), SetMode.REMOVE_IF_NULL);
+
+    if (!DataTemplateUtil.areEqual(latestSystemMetadata, previous)
+        && !DataTemplateUtil.areEqual(latestSystemMetadata, database)) {
+
+      conditionalLogLevel(
+          txContext,
+          String.format(
+              "systemMetdataDiff urn %s, %s != %s AND %s",
+              urn,
+              RecordUtils.toJsonString(latestSystemMetadata),
+              previous == null ? null : RecordUtils.toJsonString(previous),
+              database == null ? null : RecordUtils.toJsonString(database)));
+
+      return Optional.of(latestSystemMetadata);
+    }
+
+    return Optional.empty();
+  }
+
+  private static void conditionalLogLevel(@Nullable TransactionContext txContext, String message) {
+    if (txContext != null && txContext.getFailedAttempts() > 1) {
+      log.warn(message);
+    } else {
+      log.debug(message);
+    }
+  }
 }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java
index 3c4109970e9d0..da48a2b76d6d5 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java
@@ -72,7 +72,7 @@ public static void ingestChangeProposals(
     entityService.ingestProposal(
         opContext,
         AspectsBatchImpl.builder()
-            .mcps(changes, getAuditStamp(actor), opContext.getRetrieverContext().get())
+            .mcps(changes, getAuditStamp(actor), opContext.getRetrieverContext())
             .build(),
         async);
   }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/TransactionContext.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/TransactionContext.java
index 69f2f1c8981c0..6897c9152e9a2 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/TransactionContext.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/TransactionContext.java
@@ -66,4 +66,10 @@ public void commitAndContinue() {
     }
     success();
   }
+
+  public void flush() {
+    if (tx != null) {
+      tx.flush();
+    }
+  }
 }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java
index 9e7387947a954..4d177d50ea44d 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java
@@ -198,7 +198,7 @@ public void saveAspect(
   @Override
   @Nonnull
   public Map<EntityAspectIdentifier, EntityAspect> batchGet(
-      @Nonnull final Set<EntityAspectIdentifier> keys) {
+      @Nonnull final Set<EntityAspectIdentifier> keys, boolean forUpdate) {
     validateConnection();
     return keys.stream()
         .map(this::getAspect)
@@ -590,7 +590,7 @@ public long saveLatestAspect(
     // Save oldValue as the largest version + 1
     long largestVersion = ASPECT_LATEST_VERSION;
     BatchStatement batch = BatchStatement.newInstance(BatchType.UNLOGGED);
-    if (oldAspectMetadata != null && oldTime != null) {
+    if (!ASPECT_LATEST_VERSION.equals(nextVersion) && oldTime != null) {
       largestVersion = nextVersion;
       final EntityAspect aspect =
           new EntityAspect(
@@ -616,7 +616,7 @@ public long saveLatestAspect(
             newTime,
             newActor,
             newImpersonator);
-    batch = batch.add(generateSaveStatement(aspect, oldAspectMetadata == null));
+    batch = batch.add(generateSaveStatement(aspect, ASPECT_LATEST_VERSION.equals(nextVersion)));
     _cqlSession.execute(batch);
     return largestVersion;
   }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraRetentionService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraRetentionService.java
index ccc1910ba5cdb..c595e3e07b834 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraRetentionService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraRetentionService.java
@@ -64,7 +64,7 @@ protected AspectsBatch buildAspectsBatch(
       List<MetadataChangeProposal> mcps,
       @Nonnull AuditStamp auditStamp) {
     return AspectsBatchImpl.builder()
-        .mcps(mcps, auditStamp, opContext.getRetrieverContext().get())
+        .mcps(mcps, auditStamp, opContext.getRetrieverContext())
         .build();
   }
 
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java
index 6233bf5e0e35c..ea580a97c5188 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java
@@ -68,7 +68,10 @@
 
 @Slf4j
 public class EbeanAspectDao implements AspectDao, AspectMigrationsDao {
-
+  // READ COMMITED is used in conjunction with SELECT FOR UPDATE (read lock) in order
+  // to ensure that the aspect's version is not modified outside the transaction.
+  // We rely on the retry mechanism if the row is modified and will re-read (require the lock)
+  public static final TxIsolation TX_ISOLATION = TxIsolation.READ_COMMITED;
   private final Database _server;
   private boolean _connectionValidated = false;
   private final Clock _clock = Clock.systemUTC();
@@ -90,8 +93,14 @@ public class EbeanAspectDao implements AspectDao, AspectMigrationsDao {
    */
   private final LoadingCache<String, Lock> locks;
 
+  private final String batchGetMethod;
+
   public EbeanAspectDao(@Nonnull final Database server, EbeanConfiguration ebeanConfiguration) {
     _server = server;
+    this.batchGetMethod =
+        ebeanConfiguration.getBatchGetMethod() != null
+            ? ebeanConfiguration.getBatchGetMethod()
+            : "IN";
     if (ebeanConfiguration.getLocking().isEnabled()) {
       this.locks =
           CacheBuilder.newBuilder()
@@ -165,7 +174,7 @@ public long saveLatestAspect(
     }
     // Save oldValue as the largest version + 1
     long largestVersion = ASPECT_LATEST_VERSION;
-    if (oldAspectMetadata != null && oldTime != null) {
+    if (!ASPECT_LATEST_VERSION.equals(nextVersion) && oldTime != null) {
       largestVersion = nextVersion;
       saveAspect(
           txContext,
@@ -191,7 +200,7 @@ public long saveLatestAspect(
         newTime,
         newSystemMetadata,
         ASPECT_LATEST_VERSION,
-        oldAspectMetadata == null);
+        ASPECT_LATEST_VERSION.equals(nextVersion));
 
     return largestVersion;
   }
@@ -329,7 +338,7 @@ public int deleteUrn(@Nullable TransactionContext txContext, @Nonnull final Stri
   @Override
   @Nonnull
   public Map<EntityAspectIdentifier, EntityAspect> batchGet(
-      @Nonnull final Set<EntityAspectIdentifier> keys) {
+      @Nonnull final Set<EntityAspectIdentifier> keys, boolean forUpdate) {
     validateConnection();
     if (keys.isEmpty()) {
       return Collections.emptyMap();
@@ -341,9 +350,9 @@ public Map<EntityAspectIdentifier, EntityAspect> batchGet(
             .collect(Collectors.toSet());
     final List<EbeanAspectV2> records;
     if (_queryKeysCount == 0) {
-      records = batchGet(ebeanKeys, ebeanKeys.size());
+      records = batchGet(ebeanKeys, ebeanKeys.size(), forUpdate);
     } else {
-      records = batchGet(ebeanKeys, _queryKeysCount);
+      records = batchGet(ebeanKeys, _queryKeysCount, forUpdate);
     }
     return records.stream()
         .collect(
@@ -357,33 +366,48 @@ record -> record.getKey().toAspectIdentifier(), EbeanAspectV2::toEntityAspect));
    *
    * @param keys a set of keys with urn, aspect and version
    * @param keysCount the max number of keys for each sub query
+   * @param forUpdate whether the operation is intending to write to this row in a tx
    */
   @Nonnull
   private List<EbeanAspectV2> batchGet(
-      @Nonnull final Set<EbeanAspectV2.PrimaryKey> keys, final int keysCount) {
+      @Nonnull final Set<EbeanAspectV2.PrimaryKey> keys, final int keysCount, boolean forUpdate) {
     validateConnection();
 
     int position = 0;
 
     final int totalPageCount = QueryUtils.getTotalPageCount(keys.size(), keysCount);
     final List<EbeanAspectV2> finalResult =
-        batchGetUnion(new ArrayList<>(keys), keysCount, position);
+        batchGetSelectString(new ArrayList<>(keys), keysCount, position, forUpdate);
 
     while (QueryUtils.hasMore(position, keysCount, totalPageCount)) {
       position += keysCount;
       final List<EbeanAspectV2> oneStatementResult =
-          batchGetUnion(new ArrayList<>(keys), keysCount, position);
+          batchGetSelectString(new ArrayList<>(keys), keysCount, position, forUpdate);
       finalResult.addAll(oneStatementResult);
     }
 
     return finalResult;
   }
 
+  @Nonnull
+  private List<EbeanAspectV2> batchGetSelectString(
+      @Nonnull final List<EbeanAspectV2.PrimaryKey> keys,
+      final int keysCount,
+      final int position,
+      boolean forUpdate) {
+
+    if (batchGetMethod.equals("IN")) {
+      return batchGetIn(keys, keysCount, position, forUpdate);
+    }
+
+    return batchGetUnion(keys, keysCount, position, forUpdate);
+  }
+
   /**
    * Builds a single SELECT statement for batch get, which selects one entity, and then can be
    * UNION'd with other SELECT statements.
    */
-  private String batchGetSelect(
+  private String batchGetSelectString(
       final int selectId,
       @Nonnull final String urn,
       @Nonnull final String aspect,
@@ -407,7 +431,10 @@ private String batchGetSelect(
 
   @Nonnull
   private List<EbeanAspectV2> batchGetUnion(
-      @Nonnull final List<EbeanAspectV2.PrimaryKey> keys, final int keysCount, final int position) {
+      @Nonnull final List<EbeanAspectV2.PrimaryKey> keys,
+      final int keysCount,
+      final int position,
+      boolean forUpdate) {
     validateConnection();
 
     // Build one SELECT per key and then UNION ALL the results. This can be much more performant
@@ -427,7 +454,7 @@ private List<EbeanAspectV2> batchGetUnion(
     final Map<String, Object> params = new HashMap<>();
     for (int index = position; index < end; index++) {
       sb.append(
-          batchGetSelect(
+          batchGetSelectString(
               index - position,
               keys.get(index).getUrn(),
               keys.get(index).getAspect(),
@@ -439,6 +466,70 @@ private List<EbeanAspectV2> batchGetUnion(
       }
     }
 
+    // Add FOR UPDATE clause only once at the end of the entire statement
+    if (forUpdate) {
+      sb.append(" FOR UPDATE");
+    }
+
+    final RawSql rawSql =
+        RawSqlBuilder.parse(sb.toString())
+            .columnMapping(EbeanAspectV2.URN_COLUMN, "key.urn")
+            .columnMapping(EbeanAspectV2.ASPECT_COLUMN, "key.aspect")
+            .columnMapping(EbeanAspectV2.VERSION_COLUMN, "key.version")
+            .create();
+
+    final Query<EbeanAspectV2> query = _server.find(EbeanAspectV2.class).setRawSql(rawSql);
+
+    for (Map.Entry<String, Object> param : params.entrySet()) {
+      query.setParameter(param.getKey(), param.getValue());
+    }
+
+    return query.findList();
+  }
+
+  @Nonnull
+  private List<EbeanAspectV2> batchGetIn(
+      @Nonnull final List<EbeanAspectV2.PrimaryKey> keys,
+      final int keysCount,
+      final int position,
+      boolean forUpdate) {
+    validateConnection();
+
+    // Build a single SELECT with IN clause using composite key comparison
+    // Query will look like:
+    // SELECT * FROM metadata_aspect WHERE (urn, aspect, version) IN
+    // (('urn0', 'aspect0', 0), ('urn1', 'aspect1', 1))
+    final StringBuilder sb = new StringBuilder();
+    sb.append(
+        "SELECT urn, aspect, version, metadata, systemMetadata, createdOn, createdBy, createdFor ");
+    sb.append("FROM metadata_aspect_v2 WHERE (urn, aspect, version) IN (");
+
+    final int end = Math.min(keys.size(), position + keysCount);
+    final Map<String, Object> params = new HashMap<>();
+
+    for (int index = position; index < end; index++) {
+      int paramIndex = index - position;
+      String urnParam = "urn" + paramIndex;
+      String aspectParam = "aspect" + paramIndex;
+      String versionParam = "version" + paramIndex;
+
+      params.put(urnParam, keys.get(index).getUrn());
+      params.put(aspectParam, keys.get(index).getAspect());
+      params.put(versionParam, keys.get(index).getVersion());
+
+      sb.append("(:" + urnParam + ", :" + aspectParam + ", :" + versionParam + ")");
+
+      if (index != end - 1) {
+        sb.append(",");
+      }
+    }
+
+    sb.append(")");
+
+    if (forUpdate) {
+      sb.append(" FOR UPDATE");
+    }
+
     final RawSql rawSql =
         RawSqlBuilder.parse(sb.toString())
             .columnMapping(EbeanAspectV2.URN_COLUMN, "key.urn")
@@ -736,8 +827,7 @@ public <T> T runInTransactionWithRetryUnlocked(
     T result = null;
     do {
       try (Transaction transaction =
-          _server.beginTransaction(
-              TxScope.requiresNew().setIsolation(TxIsolation.REPEATABLE_READ))) {
+          _server.beginTransaction(TxScope.requiresNew().setIsolation(TX_ISOLATION))) {
         transaction.setBatchMode(true);
         result = block.apply(transactionContext.tx(transaction));
         transaction.commit();
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanRetentionService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanRetentionService.java
index 49fa555e006f6..74d0d8b0964de 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanRetentionService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanRetentionService.java
@@ -59,7 +59,7 @@ protected AspectsBatch buildAspectsBatch(
       List<MetadataChangeProposal> mcps,
       @Nonnull AuditStamp auditStamp) {
     return AspectsBatchImpl.builder()
-        .mcps(mcps, auditStamp, opContext.getRetrieverContext().get())
+        .mcps(mcps, auditStamp, opContext.getRetrieverContext())
         .build();
   }
 
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java
index 1cd738656d972..ecded1bb9c384 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java
@@ -13,6 +13,7 @@
 import com.linkedin.metadata.utils.metrics.MetricUtils;
 import io.datahubproject.metadata.context.OperationContext;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -213,7 +214,7 @@ public SearchResult searchAcrossEntities(
    * @return some entities to search
    */
   public List<String> getEntitiesToSearch(
-      @Nonnull OperationContext opContext, @Nonnull List<String> inputEntities, int size) {
+      @Nonnull OperationContext opContext, @Nonnull Collection<String> inputEntities, int size) {
     List<String> nonEmptyEntities;
     List<String> lowercaseEntities =
         inputEntities.stream().map(String::toLowerCase).collect(Collectors.toList());
@@ -247,7 +248,7 @@ public List<String> getEntitiesToSearch(
   @Nonnull
   public ScrollResult scrollAcrossEntities(
       @Nonnull OperationContext opContext,
-      @Nonnull List<String> entities,
+      @Nonnull Collection<String> entities,
       @Nonnull String input,
       @Nullable Filter postFilters,
       List<SortCriterion> sortCriteria,
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
index 6001e2f6e660f..7f1d467dbd491 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
@@ -106,17 +106,17 @@ public void deleteDocument(
 
   @Override
   public void appendRunId(
-      @Nonnull OperationContext opContext,
-      @Nonnull String entityName,
-      @Nonnull Urn urn,
-      @Nullable String runId) {
+      @Nonnull OperationContext opContext, @Nonnull Urn urn, @Nullable String runId) {
     final String docId = indexBuilders.getIndexConvention().getEntityDocumentId(urn);
 
-    log.info(
-        "Appending run id for entity name: {}, doc id: {}, run id: {}", entityName, docId, runId);
+    log.debug(
+        "Appending run id for entity name: {}, doc id: {}, run id: {}",
+        urn.getEntityType(),
+        docId,
+        runId);
     esWriteDAO.applyScriptUpdate(
         opContext,
-        entityName,
+        urn.getEntityType(),
         docId,
         /*
           Script used to apply updates to the runId field of the index.
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java
index 61bba11098fae..35f133cc794f2 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java
@@ -36,6 +36,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -425,7 +426,10 @@ public List<String> getBrowsePaths(
     if (!sourceMap.containsKey(BROWSE_PATH)) {
       return Collections.emptyList();
     }
-    return (List<String>) sourceMap.get(BROWSE_PATH);
+    List<String> browsePaths =
+        ((List<String>) sourceMap.get(BROWSE_PATH))
+            .stream().filter(Objects::nonNull).collect(Collectors.toList());
+    return browsePaths;
   }
 
   public BrowseResultV2 browseV2(
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
index f09a81c0c8b89..2d7db075e676f 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
@@ -370,7 +370,7 @@ public AutoCompleteResult autoComplete(
       IndexConvention indexConvention = opContext.getSearchContext().getIndexConvention();
       AutocompleteRequestHandler builder =
           AutocompleteRequestHandler.getBuilder(
-              entitySpec, customSearchConfiguration, queryFilterRewriteChain);
+              entitySpec, customSearchConfiguration, queryFilterRewriteChain, searchConfiguration);
       SearchRequest req =
           builder.getSearchRequest(
               opContext,
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java
index d545f60a1ee8f..6c5c6243d3362 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java
@@ -110,12 +110,9 @@ private BoolQueryBuilder handleNestedFilters(
     mustNotQueryBuilders.forEach(expandedQueryBuilder::mustNot);
     expandedQueryBuilder.queryName(boolQueryBuilder.queryName());
     expandedQueryBuilder.adjustPureNegative(boolQueryBuilder.adjustPureNegative());
+    expandedQueryBuilder.minimumShouldMatch(boolQueryBuilder.minimumShouldMatch());
     expandedQueryBuilder.boost(boolQueryBuilder.boost());
 
-    if (!expandedQueryBuilder.should().isEmpty()) {
-      expandedQueryBuilder.minimumShouldMatch(1);
-    }
-
     return expandedQueryBuilder;
   }
 
@@ -146,7 +143,7 @@ private static QueryBuilder expandTerms(
     if (!queryUrns.isEmpty()) {
 
       scrollGraph(
-          opContext.getRetrieverContext().get().getGraphRetriever(),
+          opContext.getRetrieverContext().getGraphRetriever(),
           queryUrns,
           relationshipTypes,
           relationshipDirection,
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java
index 39f69ed1716ab..8b83439a3008c 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java
@@ -5,6 +5,7 @@
 import static com.linkedin.metadata.search.utils.ESUtils.toParentField;
 import static com.linkedin.metadata.utils.SearchUtil.*;
 
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.LongMap;
 import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.config.search.SearchConfiguration;
@@ -22,10 +23,13 @@
 import com.linkedin.util.Pair;
 import io.datahubproject.metadata.context.OperationContext;
 import io.opentelemetry.extension.annotations.WithSpan;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -48,6 +52,7 @@
 @Slf4j
 public class AggregationQueryBuilder {
   private static final String URN_FILTER = "urn";
+  private static final String STRUCTURED_PROPERTIES_PREFIX = "structuredProperties.";
   private final SearchConfiguration configs;
   private final Set<String> defaultFacetFields;
   private final Set<String> allFacetFields;
@@ -80,12 +85,13 @@ public List<AggregationBuilder> getAggregations(@Nonnull OperationContext opCont
    */
   public List<AggregationBuilder> getAggregations(
       @Nonnull OperationContext opContext, @Nullable List<String> facets) {
-    final Set<String> facetsToAggregate;
+    final Set<String> facetsToAggregate = new HashSet<>();
+    if (Boolean.TRUE.equals(
+        opContext.getSearchContext().getSearchFlags().isIncludeDefaultFacets())) {
+      facetsToAggregate.addAll(defaultFacetFields);
+    }
     if (facets != null) {
-      facetsToAggregate =
-          facets.stream().filter(this::isValidAggregate).collect(Collectors.toSet());
-    } else {
-      facetsToAggregate = defaultFacetFields;
+      facets.stream().filter(this::isValidAggregate).forEach(facetsToAggregate::add);
     }
     return facetsToAggregate.stream()
         .map(f -> facetToAggregationBuilder(opContext, f))
@@ -247,7 +253,7 @@ List<AggregationMetadata> extractAggregationMetadata(
     return addFiltersToAggregationMetadata(aggregationMetadataList, filter, aspectRetriever);
   }
 
-  private void processTermAggregations(
+  public void processTermAggregations(
       final Map.Entry<String, Aggregation> entry,
       final List<AggregationMetadata> aggregationMetadataList) {
     final Map<String, Long> oneTermAggResult =
@@ -264,6 +270,7 @@ private void processTermAggregations(
             .setFilterValues(
                 new FilterValueArray(
                     SearchUtil.convertToFilters(oneTermAggResult, Collections.emptySet())));
+    updateAggregationEntity(aggregationMetadata);
     aggregationMetadataList.add(aggregationMetadata);
   }
 
@@ -300,7 +307,15 @@ private static void recurseTermsAgg(
 
   private static void processTermBucket(
       Terms.Bucket bucket, Map<String, Long> aggResult, boolean includeZeroes) {
-    String key = bucket.getKeyAsString();
+    final String key = bucket.getKeyAsString();
+    String finalKey = key;
+    try {
+      // if the value is a date string, convert to milliseconds since epoch
+      OffsetDateTime time = OffsetDateTime.parse(key);
+      finalKey = String.valueOf(time.toEpochSecond() * 1000);
+    } catch (DateTimeParseException e) {
+      // do nothing, this is expected if the value is not a date
+    }
     // Gets filtered sub aggregation doc count if exist
     Map<String, Long> subAggs = recursivelyAddNestedSubAggs(bucket.getAggregations());
     subAggs.forEach(
@@ -309,7 +324,7 @@ private static void processTermBucket(
                 String.format("%s%s%s", key, AGGREGATION_SEPARATOR_CHAR, entryKey), entryValue));
     long docCount = bucket.getDocCount();
     if (includeZeroes || docCount > 0) {
-      aggResult.put(key, docCount);
+      aggResult.put(finalKey, docCount);
     }
   }
 
@@ -379,7 +394,7 @@ private void addCriteriaFiltersToAggregationMetadata(
     }
   }
 
-  private void addCriterionFiltersToAggregationMetadata(
+  public void addCriterionFiltersToAggregationMetadata(
       @Nonnull final Criterion criterion,
       @Nonnull final List<AggregationMetadata> aggregationMetadata,
       @Nullable AspectRetriever aspectRetriever) {
@@ -422,6 +437,17 @@ private void addCriterionFiltersToAggregationMetadata(
                 value ->
                     addMissingAggregationValueToAggregationMetadata(value, originalAggMetadata));
       }
+    } else if (aggregationMetadataMap.containsKey(criterion.getField())) {
+      /*
+       * If we already have aggregations for the facet field (original field name), simply inject any missing values counts into the set.
+       * If there are no results for a particular facet value, it will NOT be in the original aggregation set returned by
+       * Elasticsearch.
+       */
+      AggregationMetadata originalAggMetadata = aggregationMetadataMap.get(criterion.getField());
+      criterion
+          .getValues()
+          .forEach(
+              value -> addMissingAggregationValueToAggregationMetadata(value, originalAggMetadata));
     } else {
       /*
        * If we do not have ANY aggregation for the facet field, then inject a new aggregation metadata object for the
@@ -429,10 +455,14 @@ private void addCriterionFiltersToAggregationMetadata(
        * If there are no results for a particular facet, it will NOT be in the original aggregation set returned by
        * Elasticsearch.
        */
+      // Simply replace suffix from original field when there are no aggregations for it. Prevents
+      // bug where ES mappings for field are different from how we map the field back to UI
+      // (ie. Structured Properties with dots in them)
+      String facetField = ESUtils.replaceSuffix(criterion.getField());
       aggregationMetadata.add(
           buildAggregationMetadata(
-              finalFacetField,
-              getFacetToDisplayNames().getOrDefault(finalFacetField, finalFacetField),
+              facetField,
+              getFacetToDisplayNames().getOrDefault(facetField, facetField),
               new LongMap(
                   criterion.getValues().stream().collect(Collectors.toMap(i -> i, i -> 0L))),
               new FilterValueArray(
@@ -459,11 +489,24 @@ private AggregationMetadata buildAggregationMetadata(
       @Nonnull final String displayName,
       @Nonnull final LongMap aggValues,
       @Nonnull final FilterValueArray filterValues) {
-    return new AggregationMetadata()
-        .setName(facetField)
-        .setDisplayName(displayName)
-        .setAggregations(aggValues)
-        .setFilterValues(filterValues);
+    AggregationMetadata aggregationMetadata =
+        new AggregationMetadata()
+            .setName(facetField)
+            .setDisplayName(displayName)
+            .setAggregations(aggValues)
+            .setFilterValues(filterValues);
+    updateAggregationEntity(aggregationMetadata);
+    return aggregationMetadata;
+  }
+
+  public void updateAggregationEntity(@Nonnull final AggregationMetadata aggregationMetadata) {
+    if (aggregationMetadata.getName().startsWith(STRUCTURED_PROPERTIES_PREFIX)) {
+      aggregationMetadata.setEntity(
+          UrnUtils.getUrn(
+              String.format(
+                  "urn:li:structuredProperty:%s",
+                  aggregationMetadata.getName().replaceFirst(STRUCTURED_PROPERTIES_PREFIX, ""))));
+    }
   }
 
   private List<Pair<String, Pair<String, String>>> getFacetFieldDisplayNameFromAnnotation(
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java
index 294efb069a904..45359285b4a04 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java
@@ -1,6 +1,5 @@
 package com.linkedin.metadata.search.elasticsearch.query.request;
 
-import static com.linkedin.metadata.models.SearchableFieldSpecExtractor.PRIMARY_URN_SEARCH_PROPERTIES;
 import static com.linkedin.metadata.search.utils.ESAccessControlUtil.restrictUrn;
 import static com.linkedin.metadata.search.utils.ESUtils.applyDefaultSearchFilters;
 
@@ -8,6 +7,7 @@
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.StringArray;
+import com.linkedin.metadata.config.search.SearchConfiguration;
 import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration;
 import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration;
 import com.linkedin.metadata.config.search.custom.QueryConfiguration;
@@ -35,6 +35,7 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Pair;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.index.query.*;
@@ -46,7 +47,7 @@
 @Slf4j
 public class AutocompleteRequestHandler {
 
-  private final List<String> _defaultAutocompleteFields;
+  private final List<Pair> _defaultAutocompleteFields;
   private final Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypes;
 
   private static final Map<EntitySpec, AutocompleteRequestHandler>
@@ -56,11 +57,13 @@ public class AutocompleteRequestHandler {
 
   private final EntitySpec entitySpec;
   private final QueryFilterRewriteChain queryFilterRewriteChain;
+  private final SearchConfiguration searchConfiguration;
 
   public AutocompleteRequestHandler(
       @Nonnull EntitySpec entitySpec,
       @Nullable CustomSearchConfiguration customSearchConfiguration,
-      @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) {
+      @Nonnull QueryFilterRewriteChain queryFilterRewriteChain,
+      @Nonnull SearchConfiguration searchConfiguration) {
     this.entitySpec = entitySpec;
     List<SearchableFieldSpec> fieldSpecs = entitySpec.getSearchableFieldSpecs();
     this.customizedQueryHandler = CustomizedQueryHandler.builder(customSearchConfiguration).build();
@@ -69,8 +72,12 @@ public AutocompleteRequestHandler(
                 fieldSpecs.stream()
                     .map(SearchableFieldSpec::getSearchableAnnotation)
                     .filter(SearchableAnnotation::isEnableAutocomplete)
-                    .map(SearchableAnnotation::getFieldName),
-                Stream.of("urn"))
+                    .map(
+                        searchableAnnotation ->
+                            Pair.of(
+                                searchableAnnotation.getFieldName(),
+                                Double.toString(searchableAnnotation.getBoostScore()))),
+                Stream.of(Pair.of("urn", "1.0")))
             .collect(Collectors.toList());
     searchableFieldTypes =
         fieldSpecs.stream()
@@ -87,17 +94,22 @@ public AutocompleteRequestHandler(
                       return set1;
                     }));
     this.queryFilterRewriteChain = queryFilterRewriteChain;
+    this.searchConfiguration = searchConfiguration;
   }
 
   public static AutocompleteRequestHandler getBuilder(
       @Nonnull EntitySpec entitySpec,
       @Nullable CustomSearchConfiguration customSearchConfiguration,
-      @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) {
+      @Nonnull QueryFilterRewriteChain queryFilterRewriteChain,
+      @Nonnull SearchConfiguration searchConfiguration) {
     return AUTOCOMPLETE_QUERY_BUILDER_BY_ENTITY_NAME.computeIfAbsent(
         entitySpec,
         k ->
             new AutocompleteRequestHandler(
-                entitySpec, customSearchConfiguration, queryFilterRewriteChain));
+                entitySpec,
+                customSearchConfiguration,
+                queryFilterRewriteChain,
+                searchConfiguration));
   }
 
   public SearchRequest getSearchRequest(
@@ -169,7 +181,7 @@ private BoolQueryBuilder getQuery(
   public BoolQueryBuilder getQuery(
       @Nonnull ObjectMapper objectMapper,
       @Nullable AutocompleteConfiguration customAutocompleteConfig,
-      List<String> autocompleteFields,
+      List<Pair> autocompleteFields,
       @Nonnull String query) {
 
     BoolQueryBuilder finalQuery =
@@ -189,7 +201,7 @@ public BoolQueryBuilder getQuery(
 
   private Optional<QueryBuilder> getAutocompleteQuery(
       @Nullable AutocompleteConfiguration customConfig,
-      List<String> autocompleteFields,
+      List<Pair> autocompleteFields,
       @Nonnull String query) {
     Optional<QueryBuilder> result = Optional.empty();
 
@@ -200,33 +212,39 @@ private Optional<QueryBuilder> getAutocompleteQuery(
     return result;
   }
 
-  private static BoolQueryBuilder defaultQuery(
-      List<String> autocompleteFields, @Nonnull String query) {
+  private BoolQueryBuilder defaultQuery(List<Pair> autocompleteFields, @Nonnull String query) {
     BoolQueryBuilder finalQuery = QueryBuilders.boolQuery().minimumShouldMatch(1);
 
     // Search for exact matches with higher boost and ngram matches
-    MultiMatchQueryBuilder autocompleteQueryBuilder =
+    MultiMatchQueryBuilder multiMatchQueryBuilder =
         QueryBuilders.multiMatchQuery(query).type(MultiMatchQueryBuilder.Type.BOOL_PREFIX);
 
-    final float urnBoost =
-        Float.parseFloat((String) PRIMARY_URN_SEARCH_PROPERTIES.get("boostScore"));
     autocompleteFields.forEach(
-        fieldName -> {
-          if ("urn".equals(fieldName)) {
-            autocompleteQueryBuilder.field(fieldName + ".ngram", urnBoost);
-            autocompleteQueryBuilder.field(fieldName + ".ngram._2gram", urnBoost);
-            autocompleteQueryBuilder.field(fieldName + ".ngram._3gram", urnBoost);
-            autocompleteQueryBuilder.field(fieldName + ".ngram._4gram", urnBoost);
-          } else {
-            autocompleteQueryBuilder.field(fieldName + ".ngram");
-            autocompleteQueryBuilder.field(fieldName + ".ngram._2gram");
-            autocompleteQueryBuilder.field(fieldName + ".ngram._3gram");
-            autocompleteQueryBuilder.field(fieldName + ".ngram._4gram");
+        pair -> {
+          final String fieldName = (String) pair.getLeft();
+          final float boostScore = Float.parseFloat((String) pair.getRight());
+          multiMatchQueryBuilder.field(fieldName + ".ngram");
+          multiMatchQueryBuilder.field(fieldName + ".ngram._2gram");
+          multiMatchQueryBuilder.field(fieldName + ".ngram._3gram");
+          multiMatchQueryBuilder.field(fieldName + ".ngram._4gram");
+          multiMatchQueryBuilder.field(fieldName + ".delimited");
+          if (!fieldName.equalsIgnoreCase("urn")) {
+            multiMatchQueryBuilder.field(fieldName + ".ngram", boostScore);
+            multiMatchQueryBuilder.field(
+                fieldName + ".ngram._2gram",
+                boostScore * (searchConfiguration.getWordGram().getTwoGramFactor()));
+            multiMatchQueryBuilder.field(
+                fieldName + ".ngram._3gram",
+                boostScore * (searchConfiguration.getWordGram().getThreeGramFactor()));
+            multiMatchQueryBuilder.field(
+                fieldName + ".ngram._4gram",
+                boostScore * (searchConfiguration.getWordGram().getFourGramFactor()));
+            finalQuery.should(
+                QueryBuilders.matchQuery(fieldName + ".keyword", query).boost(boostScore));
           }
-          autocompleteQueryBuilder.field(fieldName + ".delimited");
           finalQuery.should(QueryBuilders.matchPhrasePrefixQuery(fieldName + ".delimited", query));
         });
-    finalQuery.should(autocompleteQueryBuilder);
+    finalQuery.should(multiMatchQueryBuilder);
     return finalQuery;
   }
 
@@ -241,12 +259,17 @@ private HighlightBuilder getHighlights(@Nullable String field) {
     // Check for each field name and any subfields
     getAutocompleteFields(field)
         .forEach(
-            fieldName ->
-                highlightBuilder
-                    .field(fieldName)
-                    .field(fieldName + ".*")
-                    .field(fieldName + ".ngram")
-                    .field(fieldName + ".delimited"));
+            pair -> {
+              final String fieldName = (String) pair.getLeft();
+              highlightBuilder
+                  .field(fieldName)
+                  .field(fieldName + ".*")
+                  .field(fieldName + ".ngram")
+                  .field(fieldName + ".delimited");
+              if (!fieldName.equalsIgnoreCase("urn")) {
+                highlightBuilder.field(fieldName + ".keyword");
+              }
+            });
 
     // set field match req false for ngram
     highlightBuilder.fields().stream()
@@ -256,9 +279,9 @@ private HighlightBuilder getHighlights(@Nullable String field) {
     return highlightBuilder;
   }
 
-  private List<String> getAutocompleteFields(@Nullable String field) {
-    if (field != null && !field.isEmpty()) {
-      return ImmutableList.of(field);
+  private List<Pair> getAutocompleteFields(@Nullable String field) {
+    if (field != null && !field.isEmpty() && !field.equalsIgnoreCase("urn")) {
+      return ImmutableList.of(Pair.of(field, "10.0"));
     }
     return _defaultAutocompleteFields;
   }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java
index 4bb8e0630de48..b4ad847cb7afc 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java
@@ -437,8 +437,6 @@ private void setStructuredPropertiesSearchValue(
 
     Map<Urn, Map<String, Aspect>> definitions =
         opContext
-            .getRetrieverContext()
-            .get()
             .getAspectRetriever()
             .getLatestAspectObjects(
                 propertyMap.keySet(), Set.of(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME));
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
index f72b5fc1f6d22..17bbbaf059dec 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
@@ -100,7 +100,7 @@ public class ESUtils {
   // top-level properties
   // to field level properties
   public static final Map<String, List<String>> FIELDS_TO_EXPANDED_FIELDS_LIST =
-      new HashMap<String, List<String>>() {
+      new HashMap<>() {
         {
           put("tags", ImmutableList.of("tags", "fieldTags", "editedFieldTags"));
           put(
@@ -117,6 +117,8 @@ public class ESUtils {
           put(
               "businessAttribute",
               ImmutableList.of("businessAttributeRef", "businessAttributeRef.urn"));
+          put("origin", ImmutableList.of("origin", "env"));
+          put("env", ImmutableList.of("env", "origin"));
         }
       };
 
@@ -446,9 +448,20 @@ public static String toParentField(
                             urnDefinition.getFirst(), urnDefinition.getSecond()))
             .orElse(filterField);
 
+    return replaceSuffix(fieldName);
+  }
+
+  /**
+   * Strip subfields from filter field
+   *
+   * @param fieldName name of the field
+   * @return normalized field name without subfields
+   */
+  @Nonnull
+  public static String replaceSuffix(@Nonnull final String fieldName) {
     for (String subfield : SUBFIELDS) {
       String SUFFIX = "." + subfield;
-      if (filterField.endsWith(SUFFIX)) {
+      if (fieldName.endsWith(SUFFIX)) {
         return fieldName.replace(SUFFIX, "");
       }
     }
@@ -708,7 +721,8 @@ private static QueryBuilder buildEqualsConditionFromCriterionWithValues(
       final Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypes,
       @Nonnull AspectRetriever aspectRetriever,
       boolean enableCaseInsensitiveSearch) {
-    Set<String> fieldTypes = getFieldTypes(searchableFieldTypes, fieldName, aspectRetriever);
+    Set<String> fieldTypes =
+        getFieldTypes(searchableFieldTypes, fieldName, criterion, aspectRetriever);
     if (fieldTypes.size() > 1) {
       log.warn(
           "Multiple field types for field name {}, determining best fit for set: {}",
@@ -751,12 +765,16 @@ private static QueryBuilder buildEqualsConditionFromCriterionWithValues(
   private static Set<String> getFieldTypes(
       Map<String, Set<SearchableAnnotation.FieldType>> searchableFields,
       String fieldName,
+      @Nonnull final Criterion criterion,
       @Nullable AspectRetriever aspectRetriever) {
 
     final Set<String> finalFieldTypes;
     if (fieldName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) {
+      // use criterion field here for structured props since fieldName has dots replaced with
+      // underscores
       finalFieldTypes =
-          StructuredPropertyUtils.toElasticsearchFieldType(fieldName, aspectRetriever);
+          StructuredPropertyUtils.toElasticsearchFieldType(
+              replaceSuffix(criterion.getField()), aspectRetriever);
     } else {
       Set<SearchableAnnotation.FieldType> fieldTypes =
           searchableFields.getOrDefault(fieldName.split("\\.")[0], Collections.emptySet());
@@ -780,7 +798,8 @@ private static RangeQueryBuilder buildRangeQueryFromCriterion(
       Condition condition,
       boolean isTimeseries,
       AspectRetriever aspectRetriever) {
-    Set<String> fieldTypes = getFieldTypes(searchableFieldTypes, fieldName, aspectRetriever);
+    Set<String> fieldTypes =
+        getFieldTypes(searchableFieldTypes, fieldName, criterion, aspectRetriever);
 
     // Determine criterion value, range query only accepts single value so take first value in
     // values if multiple
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java
index ad2825ead3d0d..4a692e9534622 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java
@@ -112,7 +112,7 @@ private void fetchRelatedEntities(
       @Nullable String scrollId,
       int consumedEntityCount,
       int batchNumber) {
-    GraphRetriever graph = opContext.getRetrieverContext().get().getGraphRetriever();
+    GraphRetriever graph = opContext.getRetrieverContext().getGraphRetriever();
     final ArrayList<Future<ExecutionResult>> futureList = new ArrayList<>();
     RelatedEntitiesScrollResult result =
         graph.scrollRelatedEntities(
@@ -165,7 +165,7 @@ private Callable<ExecutionResult> processBatch(
     return () -> {
       StopWatch stopWatch = new StopWatch();
       stopWatch.start();
-      AspectRetriever aspectRetriever = opContext.getAspectRetrieverOpt().get();
+      AspectRetriever aspectRetriever = opContext.getAspectRetriever();
       log.info("Batch {} for BA:{} started", batchNumber, entityKey);
       ExecutionResult executionResult = new ExecutionResult();
       executionResult.setBatchNumber(batchNumber);
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java
index 7549aea2007da..4b09bc00efb61 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java
@@ -34,6 +34,7 @@
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.query.filter.RelationshipDirection;
 import com.linkedin.metadata.utils.SchemaFieldUtils;
+import com.linkedin.metadata.utils.metrics.MetricUtils;
 import com.linkedin.mxe.MetadataChangeLog;
 import com.linkedin.mxe.SystemMetadata;
 import com.linkedin.util.Pair;
@@ -56,6 +57,9 @@
 @Slf4j
 public class UpdateGraphIndicesService implements SearchIndicesService {
   private static final String DOWNSTREAM_OF = "DownstreamOf";
+  private static final String GRAPH_DIFF_MODE_REMOVE_METRIC = "diff_remove_edge";
+  private static final String GRAPH_DIFF_MODE_ADD_METRIC = "diff_add_edge";
+  private static final String GRAPH_DIFF_MODE_UPDATE_METRIC = "diff_update_edge";
 
   public static UpdateGraphIndicesService withService(GraphService graphService) {
     return new UpdateGraphIndicesService(graphService);
@@ -90,8 +94,7 @@ public UpdateGraphIndicesService(
   public void handleChangeEvent(
       @Nonnull OperationContext opContext, @Nonnull final MetadataChangeLog event) {
     try {
-      MCLItemImpl mclItem =
-          MCLItemImpl.builder().build(event, opContext.getAspectRetrieverOpt().get());
+      MCLItemImpl mclItem = MCLItemImpl.builder().build(event, opContext.getAspectRetriever());
 
       if (UPDATE_CHANGE_TYPES.contains(event.getChangeType())) {
         handleUpdateChangeEvent(opContext, mclItem);
@@ -186,7 +189,10 @@ private void handleDeleteChangeEvent(
               urn.getEntityType(), event.getAspectName()));
     }
 
-    RecordTemplate aspect = event.getRecordTemplate();
+    final RecordTemplate aspect =
+        event.getPreviousRecordTemplate() != null
+            ? event.getPreviousRecordTemplate()
+            : event.getRecordTemplate();
     Boolean isDeletingKey = event.getAspectName().equals(entitySpec.getKeyAspectName());
 
     if (!aspectSpec.isTimeseries()) {
@@ -276,8 +282,8 @@ private Pair<List<Edge>, HashMap<Urn, Set<String>>> getEdgesAndRelationshipTypes
       @Nonnull final RecordTemplate aspect,
       @Nonnull final MetadataChangeLog event,
       final boolean isNewAspectVersion) {
-    final List<Edge> edgesToAdd = new ArrayList<>();
-    final HashMap<Urn, Set<String>> urnToRelationshipTypesBeingAdded = new HashMap<>();
+    final List<Edge> edges = new ArrayList<>();
+    final HashMap<Urn, Set<String>> urnToRelationshipTypes = new HashMap<>();
 
     // we need to manually set schemaField <-> schemaField edges for fineGrainedLineage and
     // inputFields
@@ -285,36 +291,28 @@ private Pair<List<Edge>, HashMap<Urn, Set<String>>> getEdgesAndRelationshipTypes
     if (aspectSpec.getName().equals(Constants.UPSTREAM_LINEAGE_ASPECT_NAME)) {
       UpstreamLineage upstreamLineage = new UpstreamLineage(aspect.data());
       updateFineGrainedEdgesAndRelationships(
-          urn,
-          upstreamLineage.getFineGrainedLineages(),
-          edgesToAdd,
-          urnToRelationshipTypesBeingAdded);
+          urn, upstreamLineage.getFineGrainedLineages(), edges, urnToRelationshipTypes);
     } else if (aspectSpec.getName().equals(Constants.INPUT_FIELDS_ASPECT_NAME)) {
       final InputFields inputFields = new InputFields(aspect.data());
-      updateInputFieldEdgesAndRelationships(
-          urn, inputFields, edgesToAdd, urnToRelationshipTypesBeingAdded);
+      updateInputFieldEdgesAndRelationships(urn, inputFields, edges, urnToRelationshipTypes);
     } else if (aspectSpec.getName().equals(Constants.DATA_JOB_INPUT_OUTPUT_ASPECT_NAME)) {
       DataJobInputOutput dataJobInputOutput = new DataJobInputOutput(aspect.data());
       updateFineGrainedEdgesAndRelationships(
-          urn,
-          dataJobInputOutput.getFineGrainedLineages(),
-          edgesToAdd,
-          urnToRelationshipTypesBeingAdded);
+          urn, dataJobInputOutput.getFineGrainedLineages(), edges, urnToRelationshipTypes);
     }
 
     Map<RelationshipFieldSpec, List<Object>> extractedFields =
-        FieldExtractor.extractFields(aspect, aspectSpec.getRelationshipFieldSpecs());
+        FieldExtractor.extractFields(aspect, aspectSpec.getRelationshipFieldSpecs(), true);
 
     for (Map.Entry<RelationshipFieldSpec, List<Object>> entry : extractedFields.entrySet()) {
-      Set<String> relationshipTypes =
-          urnToRelationshipTypesBeingAdded.getOrDefault(urn, new HashSet<>());
+      Set<String> relationshipTypes = urnToRelationshipTypes.getOrDefault(urn, new HashSet<>());
       relationshipTypes.add(entry.getKey().getRelationshipName());
-      urnToRelationshipTypesBeingAdded.put(urn, relationshipTypes);
+      urnToRelationshipTypes.put(urn, relationshipTypes);
       final List<Edge> newEdges =
           GraphIndexUtils.extractGraphEdges(entry, aspect, urn, event, isNewAspectVersion);
-      edgesToAdd.addAll(newEdges);
+      edges.addAll(newEdges);
     }
-    return Pair.of(edgesToAdd, urnToRelationshipTypesBeingAdded);
+    return Pair.of(edges, urnToRelationshipTypes);
   }
 
   /** Process snapshot and update graph index */
@@ -382,21 +380,26 @@ private void updateGraphServiceDiff(
     final List<Edge> mergedEdges = getMergedEdges(oldEdgeSet, newEdgeSet);
 
     // Remove any old edges that no longer exist first
-    if (subtractiveDifference.size() > 0) {
+    if (!subtractiveDifference.isEmpty()) {
       log.debug("Removing edges: {}", subtractiveDifference);
       subtractiveDifference.forEach(graphService::removeEdge);
+      MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_REMOVE_METRIC)
+          .inc(subtractiveDifference.size());
     }
 
     // Then add new edges
-    if (additiveDifference.size() > 0) {
+    if (!additiveDifference.isEmpty()) {
       log.debug("Adding edges: {}", additiveDifference);
       additiveDifference.forEach(graphService::addEdge);
+      MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_ADD_METRIC)
+          .inc(additiveDifference.size());
     }
 
     // Then update existing edges
-    if (mergedEdges.size() > 0) {
+    if (!mergedEdges.isEmpty()) {
       log.debug("Updating edges: {}", mergedEdges);
       mergedEdges.forEach(graphService::upsertEdge);
+      MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_UPDATE_METRIC).inc(mergedEdges.size());
     }
   }
 
@@ -424,7 +427,7 @@ private void deleteGraphData(
       @Nonnull final OperationContext opContext,
       @Nonnull final Urn urn,
       @Nonnull final AspectSpec aspectSpec,
-      @Nonnull final RecordTemplate aspect,
+      @Nullable final RecordTemplate aspect,
       @Nonnull final Boolean isKeyAspect,
       @Nonnull final MetadataChangeLog event) {
     if (isKeyAspect) {
@@ -432,21 +435,28 @@ private void deleteGraphData(
       return;
     }
 
-    Pair<List<Edge>, HashMap<Urn, Set<String>>> edgeAndRelationTypes =
-        getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, aspect, event, true);
-
-    final HashMap<Urn, Set<String>> urnToRelationshipTypesBeingAdded =
-        edgeAndRelationTypes.getSecond();
-    if (urnToRelationshipTypesBeingAdded.size() > 0) {
-      for (Map.Entry<Urn, Set<String>> entry : urnToRelationshipTypesBeingAdded.entrySet()) {
-        graphService.removeEdgesFromNode(
-            opContext,
-            entry.getKey(),
-            new ArrayList<>(entry.getValue()),
-            createRelationshipFilter(
-                new Filter().setOr(new ConjunctiveCriterionArray()),
-                RelationshipDirection.OUTGOING));
+    if (aspect != null) {
+      Pair<List<Edge>, HashMap<Urn, Set<String>>> edgeAndRelationTypes =
+          getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, aspect, event, true);
+
+      final HashMap<Urn, Set<String>> urnToRelationshipTypesBeingRemoved =
+          edgeAndRelationTypes.getSecond();
+      if (!urnToRelationshipTypesBeingRemoved.isEmpty()) {
+        for (Map.Entry<Urn, Set<String>> entry : urnToRelationshipTypesBeingRemoved.entrySet()) {
+          graphService.removeEdgesFromNode(
+              opContext,
+              entry.getKey(),
+              new ArrayList<>(entry.getValue()),
+              createRelationshipFilter(
+                  new Filter().setOr(new ConjunctiveCriterionArray()),
+                  RelationshipDirection.OUTGOING));
+        }
       }
+    } else {
+      log.warn(
+          "Insufficient information to perform graph delete. Missing deleted aspect {} for entity {}",
+          aspectSpec.getName(),
+          urn);
     }
   }
 }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java
index 3795fd19316b1..c5fc9ebdac9fa 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java
@@ -26,6 +26,7 @@
 import com.linkedin.metadata.systemmetadata.SystemMetadataService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import com.linkedin.metadata.timeseries.transformer.TimeseriesAspectTransformer;
+import com.linkedin.metadata.utils.metrics.MetricUtils;
 import com.linkedin.mxe.MetadataChangeLog;
 import com.linkedin.mxe.SystemMetadata;
 import com.linkedin.structured.StructuredPropertyDefinition;
@@ -61,6 +62,9 @@ public class UpdateIndicesService implements SearchIndicesService {
 
   @Getter private final boolean structuredPropertiesWriteEnabled;
 
+  private static final String DOCUMENT_TRANSFORM_FAILED_METRIC = "document_transform_failed";
+  private static final String SEARCH_DIFF_MODE_SKIPPED_METRIC = "search_diff_no_changes_detected";
+
   private static final Set<ChangeType> UPDATE_CHANGE_TYPES =
       ImmutableSet.of(
           ChangeType.CREATE,
@@ -117,11 +121,10 @@ public UpdateIndicesService(
   public void handleChangeEvent(
       @Nonnull OperationContext opContext, @Nonnull final MetadataChangeLog event) {
     try {
-      MCLItemImpl batch =
-          MCLItemImpl.builder().build(event, opContext.getAspectRetrieverOpt().get());
+      MCLItemImpl batch = MCLItemImpl.builder().build(event, opContext.getAspectRetriever());
 
       Stream<MCLItem> sideEffects =
-          AspectsBatch.applyMCLSideEffects(List.of(batch), opContext.getRetrieverContext().get());
+          AspectsBatch.applyMCLSideEffects(List.of(batch), opContext.getRetrieverContext());
 
       for (MCLItem mclItem :
           Stream.concat(Stream.of(batch), sideEffects).collect(Collectors.toList())) {
@@ -176,8 +179,12 @@ private void handleUpdateChangeEvent(
       updateSystemMetadata(event.getSystemMetadata(), urn, aspectSpec, aspect);
     }
 
-    // Step 1. Handle StructuredProperties Index Mapping changes
-    updateIndexMappings(urn, entitySpec, aspectSpec, aspect, previousAspect);
+    try {
+      // Step 1. Handle StructuredProperties Index Mapping changes
+      updateIndexMappings(urn, entitySpec, aspectSpec, aspect, previousAspect);
+    } catch (Exception e) {
+      log.error("Issue with updating index mappings for structured property change", e);
+    }
 
     // Step 2. For all aspects, attempt to update Search
     updateSearchService(opContext, event);
@@ -188,7 +195,8 @@ public void updateIndexMappings(
       EntitySpec entitySpec,
       AspectSpec aspectSpec,
       RecordTemplate newValue,
-      RecordTemplate oldValue) {
+      RecordTemplate oldValue)
+      throws CloneNotSupportedException {
     if (structuredPropertiesHookEnabled
         && STRUCTURED_PROPERTY_ENTITY_NAME.equals(entitySpec.getName())
         && STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME.equals(aspectSpec.getName())) {
@@ -201,7 +209,7 @@ public void updateIndexMappings(
               .orElse(new UrnArray());
 
       StructuredPropertyDefinition newDefinition =
-          new StructuredPropertyDefinition(newValue.data());
+          new StructuredPropertyDefinition(newValue.data().copy());
       newDefinition.getEntityTypes().removeAll(oldEntityTypes);
 
       if (newDefinition.getEntityTypes().size() > 0) {
@@ -283,11 +291,13 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev
                           event.getAuditStamp()));
     } catch (Exception e) {
       log.error(
-          "Error in getting documents from aspect: {} for aspect {}", e, aspectSpec.getName());
+          "Error in getting documents for urn: {} from aspect: {}", urn, aspectSpec.getName(), e);
+      MetricUtils.counter(this.getClass(), DOCUMENT_TRANSFORM_FAILED_METRIC).inc();
       return;
     }
 
-    if (!searchDocument.isPresent()) {
+    if (searchDocument.isEmpty()) {
+      log.info("Search document for urn: {} aspect: {} was empty", urn, aspect);
       return;
     }
 
@@ -304,15 +314,22 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev
                   opContext, urn, previousAspect, aspectSpec, false);
         } catch (Exception e) {
           log.error(
-              "Error in getting documents from previous aspect state: {} for aspect {}, continuing without diffing.",
-              e,
-              aspectSpec.getName());
+              "Error in getting documents from previous aspect state for urn: {} for aspect {}, continuing without diffing.",
+              urn,
+              aspectSpec.getName(),
+              e);
+          MetricUtils.counter(this.getClass(), DOCUMENT_TRANSFORM_FAILED_METRIC).inc();
         }
       }
 
       if (previousSearchDocument.isPresent()) {
         if (searchDocument.get().toString().equals(previousSearchDocument.get().toString())) {
           // No changes to search document, skip writing no-op update
+          log.info(
+              "No changes detected for search document for urn: {} aspect: {}",
+              urn,
+              aspectSpec.getName());
+          MetricUtils.counter(this.getClass(), SEARCH_DIFF_MODE_SKIPPED_METRIC).inc();
           return;
         }
       }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffect.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffect.java
index 134c65d2b5fae..5bbc25cfc8ddc 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffect.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffect.java
@@ -3,8 +3,6 @@
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_KEY_ASPECT_NAME;
-import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX;
-import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion;
 
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
@@ -17,30 +15,19 @@
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
 import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect;
-import com.linkedin.metadata.entity.SearchRetriever;
 import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl;
 import com.linkedin.metadata.models.EntitySpec;
-import com.linkedin.metadata.models.StructuredPropertyUtils;
-import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
-import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
-import com.linkedin.metadata.query.filter.Criterion;
-import com.linkedin.metadata.query.filter.CriterionArray;
-import com.linkedin.metadata.query.filter.Filter;
-import com.linkedin.metadata.search.ScrollResult;
+import com.linkedin.metadata.structuredproperties.util.EntityWithPropertyIterator;
 import com.linkedin.structured.StructuredPropertyDefinition;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Spliterator;
 import java.util.Spliterators;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.Builder;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.experimental.Accessors;
@@ -141,60 +128,4 @@ private static Stream<MCPItem> generatePatchMCPs(
                               .build(retrieverContext.getAspectRetriever().getEntityRegistry());
                         }));
   }
-
-  /**
-   * Fetches pages of entity urns which have a value for the given structured property definition
-   */
-  @Builder
-  public static class EntityWithPropertyIterator implements Iterator<ScrollResult> {
-    @Nonnull private final Urn propertyUrn;
-    @Nullable private final StructuredPropertyDefinition definition;
-    @Nonnull private final SearchRetriever searchRetriever;
-    private int count;
-    @Builder.Default private String scrollId = null;
-    @Builder.Default private boolean started = false;
-
-    private List<String> getEntities() {
-      if (definition != null && definition.getEntityTypes() != null) {
-        return definition.getEntityTypes().stream()
-            .map(StructuredPropertyUtils::getValueTypeId)
-            .collect(Collectors.toList());
-      } else {
-        return Collections.emptyList();
-      }
-    }
-
-    private Filter getFilter() {
-      Filter propertyFilter = new Filter();
-      final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray();
-      final ConjunctiveCriterion conjunction = new ConjunctiveCriterion();
-      final CriterionArray andCriterion = new CriterionArray();
-
-      // Cannot rely on automatic field name since the definition is deleted
-      final Criterion propertyExistsCriterion =
-          buildExistsCriterion(
-              STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX
-                  + StructuredPropertyUtils.toElasticsearchFieldName(propertyUrn, definition));
-
-      andCriterion.add(propertyExistsCriterion);
-      conjunction.setAnd(andCriterion);
-      disjunction.add(conjunction);
-      propertyFilter.setOr(disjunction);
-
-      return propertyFilter;
-    }
-
-    @Override
-    public boolean hasNext() {
-      return !started || scrollId != null;
-    }
-
-    @Override
-    public ScrollResult next() {
-      started = true;
-      ScrollResult result = searchRetriever.scroll(getEntities(), getFilter(), scrollId, count);
-      scrollId = result.getScrollId();
-      return result;
-    }
-  }
 }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/util/EntityWithPropertyIterator.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/util/EntityWithPropertyIterator.java
new file mode 100644
index 0000000000000..ba501f9b43d6b
--- /dev/null
+++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/util/EntityWithPropertyIterator.java
@@ -0,0 +1,76 @@
+package com.linkedin.metadata.structuredproperties.util;
+
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX;
+import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.metadata.entity.SearchRetriever;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.Criterion;
+import com.linkedin.metadata.query.filter.CriterionArray;
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.search.ScrollResult;
+import com.linkedin.structured.StructuredPropertyDefinition;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import lombok.Builder;
+
+/** Fetches pages of entity urns which have a value for the given structured property definition */
+@Builder
+public class EntityWithPropertyIterator implements Iterator<ScrollResult> {
+  @Nonnull private final Urn propertyUrn;
+  @Nullable private final StructuredPropertyDefinition definition;
+  @Nonnull private final SearchRetriever searchRetriever;
+  private int count;
+  @Builder.Default private String scrollId = null;
+  @Builder.Default private boolean started = false;
+
+  private List<String> getEntities() {
+    if (definition != null && definition.getEntityTypes() != null) {
+      return definition.getEntityTypes().stream()
+          .map(StructuredPropertyUtils::getValueTypeId)
+          .collect(Collectors.toList());
+    } else {
+      return Collections.emptyList();
+    }
+  }
+
+  private Filter getFilter() {
+    Filter propertyFilter = new Filter();
+    final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray();
+    final ConjunctiveCriterion conjunction = new ConjunctiveCriterion();
+    final CriterionArray andCriterion = new CriterionArray();
+
+    // Cannot rely on automatic field name since the definition is deleted
+    final Criterion propertyExistsCriterion =
+        buildExistsCriterion(
+            STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX
+                + StructuredPropertyUtils.toElasticsearchFieldName(propertyUrn, definition));
+
+    andCriterion.add(propertyExistsCriterion);
+    conjunction.setAnd(andCriterion);
+    disjunction.add(conjunction);
+    propertyFilter.setOr(disjunction);
+
+    return propertyFilter;
+  }
+
+  @Override
+  public boolean hasNext() {
+    return !started || scrollId != null;
+  }
+
+  @Override
+  public ScrollResult next() {
+    started = true;
+    ScrollResult result = searchRetriever.scroll(getEntities(), getFilter(), scrollId, count);
+    scrollId = result.getScrollId();
+    return result;
+  }
+}
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/HidePropertyValidator.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/HidePropertyValidator.java
new file mode 100644
index 0000000000000..9a238d7df7750
--- /dev/null
+++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/HidePropertyValidator.java
@@ -0,0 +1,63 @@
+package com.linkedin.metadata.structuredproperties.validation;
+
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.linkedin.metadata.aspect.RetrieverContext;
+import com.linkedin.metadata.aspect.batch.BatchItem;
+import com.linkedin.metadata.aspect.batch.ChangeMCP;
+import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
+import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
+import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
+import com.linkedin.structured.StructuredPropertySettings;
+import java.util.Collection;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+
+@Setter
+@Getter
+@Slf4j
+@Accessors(chain = true)
+public class HidePropertyValidator extends AspectPayloadValidator {
+
+  @Nonnull private AspectPluginConfig config;
+
+  @Override
+  protected Stream<AspectValidationException> validateProposedAspects(
+      @Nonnull Collection<? extends BatchItem> mcpItems,
+      @Nonnull RetrieverContext retrieverContext) {
+    return validateSettingsUpserts(
+        mcpItems.stream()
+            .filter(i -> STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME.equals(i.getAspectName()))
+            .collect(Collectors.toList()));
+  }
+
+  @Override
+  protected Stream<AspectValidationException> validatePreCommitAspects(
+      @Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull RetrieverContext retrieverContext) {
+    return Stream.empty();
+  }
+
+  @VisibleForTesting
+  public static Stream<AspectValidationException> validateSettingsUpserts(
+      @Nonnull Collection<? extends BatchItem> mcpItems) {
+    ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection();
+    for (BatchItem mcpItem : mcpItems) {
+      StructuredPropertySettings structuredPropertySettings =
+          mcpItem.getAspect(StructuredPropertySettings.class);
+      boolean isValid =
+          StructuredPropertyUtils.validatePropertySettings(structuredPropertySettings, false);
+      if (!isValid) {
+        exceptions.addException(mcpItem, StructuredPropertyUtils.INVALID_SETTINGS_MESSAGE);
+      }
+    }
+    return exceptions.streamAllExceptions();
+  }
+}
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java
index ae5472af622ad..6e047c12da9a9 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java
@@ -89,6 +89,9 @@ public static Stream<AspectValidationException> validateDefinitionUpserts(
           item.getAspect(StructuredPropertyDefinition.class);
 
       versionFormatCheck(item, newDefinition.getVersion()).ifPresent(exceptions::addException);
+      urnIdCheck(item).ifPresent(exceptions::addException);
+      qualifiedNameCheck(item, newDefinition.getQualifiedName())
+          .ifPresent(exceptions::addException);
 
       if (item.getPreviousSystemAspect() != null) {
 
@@ -192,4 +195,20 @@ private static Optional<AspectValidationException> versionFormatCheck(
     }
     return Optional.empty();
   }
+
+  private static Optional<AspectValidationException> urnIdCheck(MCPItem item) {
+    if (item.getUrn().getId().contains(" ")) {
+      return Optional.of(AspectValidationException.forItem(item, "Urn ID cannot have spaces"));
+    }
+    return Optional.empty();
+  }
+
+  private static Optional<AspectValidationException> qualifiedNameCheck(
+      MCPItem item, @Nonnull String qualifiedName) {
+    if (qualifiedName.contains(" ")) {
+      return Optional.of(
+          AspectValidationException.forItem(item, "Qualified names cannot have spaces"));
+    }
+    return Optional.empty();
+  }
 }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/ShowPropertyAsBadgeValidator.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/ShowPropertyAsBadgeValidator.java
new file mode 100644
index 0000000000000..31f04e0b9c75c
--- /dev/null
+++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/ShowPropertyAsBadgeValidator.java
@@ -0,0 +1,144 @@
+package com.linkedin.metadata.structuredproperties.validation;
+
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME;
+import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion;
+
+import com.datahub.util.RecordUtils;
+import com.google.common.annotations.VisibleForTesting;
+import com.linkedin.entity.Aspect;
+import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.RetrieverContext;
+import com.linkedin.metadata.aspect.batch.BatchItem;
+import com.linkedin.metadata.aspect.batch.ChangeMCP;
+import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
+import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
+import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection;
+import com.linkedin.metadata.entity.GenericScrollIterator;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
+import com.linkedin.metadata.query.filter.Condition;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.Criterion;
+import com.linkedin.metadata.query.filter.CriterionArray;
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.search.ScrollResult;
+import com.linkedin.metadata.search.SearchEntity;
+import com.linkedin.structured.StructuredPropertySettings;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+
+@Setter
+@Getter
+@Slf4j
+@Accessors(chain = true)
+public class ShowPropertyAsBadgeValidator extends AspectPayloadValidator {
+
+  @Nonnull private AspectPluginConfig config;
+
+  private static final String SHOW_ASSET_AS_BADGE_FIELD = "showAsAssetBadge";
+
+  @Override
+  protected Stream<AspectValidationException> validateProposedAspects(
+      @Nonnull Collection<? extends BatchItem> mcpItems,
+      @Nonnull RetrieverContext retrieverContext) {
+    return validateSettingsUpserts(
+        mcpItems.stream()
+            .filter(i -> STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME.equals(i.getAspectName()))
+            .collect(Collectors.toList()),
+        retrieverContext);
+  }
+
+  @Override
+  protected Stream<AspectValidationException> validatePreCommitAspects(
+      @Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull RetrieverContext retrieverContext) {
+    return Stream.empty();
+  }
+
+  @VisibleForTesting
+  public static Stream<AspectValidationException> validateSettingsUpserts(
+      @Nonnull Collection<? extends BatchItem> mcpItems,
+      @Nonnull RetrieverContext retrieverContext) {
+    ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection();
+    for (BatchItem mcpItem : mcpItems) {
+      StructuredPropertySettings structuredPropertySettings =
+          mcpItem.getAspect(StructuredPropertySettings.class);
+      if (structuredPropertySettings.isShowAsAssetBadge()) {
+        // Search for any structured properties that have showAsAssetBadge set, should only ever be
+        // one at most.
+        GenericScrollIterator scrollIterator =
+            GenericScrollIterator.builder()
+                .searchRetriever(retrieverContext.getSearchRetriever())
+                .count(10) // Get first 10, should only ever be one, but this gives us more info if
+                // we're in a bad state
+                .filter(getFilter())
+                .entities(Collections.singletonList(STRUCTURED_PROPERTY_ENTITY_NAME))
+                .build();
+        // Only need to get first set, if there are more then will have to resolve bad state
+        ScrollResult scrollResult = scrollIterator.next();
+        if (CollectionUtils.isNotEmpty(scrollResult.getEntities())) {
+          if (scrollResult.getEntities().size() > 1) {
+            // If it's greater than one, don't bother querying DB since we for sure are in a bad
+            // state
+            exceptions.addException(
+                mcpItem,
+                StructuredPropertyUtils.ONLY_ONE_BADGE
+                    + scrollResult.getEntities().stream()
+                        .map(SearchEntity::getEntity)
+                        .collect(Collectors.toList()));
+          } else {
+            // If there is just one, verify against DB to make sure we're not hitting a timing issue
+            // with eventual consistency
+            AspectRetriever aspectRetriever = retrieverContext.getAspectRetriever();
+            Optional<Aspect> propertySettings =
+                Optional.ofNullable(
+                    aspectRetriever.getLatestAspectObject(
+                        scrollResult.getEntities().get(0).getEntity(),
+                        STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME));
+            if (propertySettings.isPresent()) {
+              StructuredPropertySettings dbBadgeSettings =
+                  RecordUtils.toRecordTemplate(
+                      StructuredPropertySettings.class, propertySettings.get().data());
+              if (dbBadgeSettings.isShowAsAssetBadge()) {
+                exceptions.addException(
+                    mcpItem,
+                    StructuredPropertyUtils.ONLY_ONE_BADGE
+                        + scrollResult.getEntities().stream()
+                            .map(SearchEntity::getEntity)
+                            .collect(Collectors.toList()));
+              }
+            }
+          }
+        }
+      }
+    }
+    return exceptions.streamAllExceptions();
+  }
+
+  private static Filter getFilter() {
+    Filter propertyFilter = new Filter();
+    final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray();
+    final ConjunctiveCriterion conjunction = new ConjunctiveCriterion();
+    final CriterionArray andCriterion = new CriterionArray();
+
+    final Criterion propertyExistsCriterion =
+        buildCriterion(SHOW_ASSET_AS_BADGE_FIELD, Condition.EQUAL, "true");
+
+    andCriterion.add(propertyExistsCriterion);
+    conjunction.setAnd(andCriterion);
+    disjunction.add(conjunction);
+    propertyFilter.setOr(disjunction);
+
+    return propertyFilter;
+  }
+}
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java
index 53f757d8d6c6b..df902f95245ef 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java
@@ -266,6 +266,7 @@ private static List<ChangeEvent> computeDiffs(
         SchemaField renamedField =
             findRenamedField(
                 curBaseField,
+                new HashSet<>(baseFields.subList(baseFieldIdx, baseFields.size())),
                 targetFields.subList(targetFieldIdx, targetFields.size()),
                 renamedFields);
         if (renamedField == null) {
@@ -289,7 +290,10 @@ private static List<ChangeEvent> computeDiffs(
         // minor version bump for both.
         SchemaField renamedField =
             findRenamedField(
-                curTargetField, baseFields.subList(baseFieldIdx, baseFields.size()), renamedFields);
+                curTargetField,
+                new HashSet<>(targetFields.subList(targetFieldIdx, targetFields.size())),
+                baseFields.subList(baseFieldIdx, baseFields.size()),
+                renamedFields);
         if (renamedField == null) {
           processAdd(changeCategories, changeEvents, datasetUrn, curTargetField, auditStamp);
           ++targetFieldIdx;
@@ -348,10 +352,14 @@ private static void sortFieldsByPath(SchemaMetadata schemaMetadata) {
   }
 
   private static SchemaField findRenamedField(
-      SchemaField curField, List<SchemaField> targetFields, Set<SchemaField> renamedFields) {
+      SchemaField curField,
+      Set<SchemaField> baseFields,
+      List<SchemaField> targetFields,
+      Set<SchemaField> renamedFields) {
     return targetFields.stream()
         .filter(schemaField -> isRenamed(curField, schemaField))
         .filter(field -> !renamedFields.contains(field))
+        .filter(field -> !baseFields.contains(field)) // Filter out fields that will match later
         .findFirst()
         .orElse(null);
   }
@@ -515,8 +523,8 @@ private static List<ChangeEvent> getPrimaryKeyChangeEvents(
       SchemaMetadata targetSchema,
       Urn datasetUrn,
       AuditStamp auditStamp) {
+    List<ChangeEvent> primaryKeyChangeEvents = new ArrayList<>();
     if (changeCategories != null && changeCategories.contains(ChangeCategory.TECHNICAL_SCHEMA)) {
-      List<ChangeEvent> primaryKeyChangeEvents = new ArrayList<>();
       Set<String> basePrimaryKeys =
           (baseSchema != null && baseSchema.getPrimaryKeys() != null)
               ? new HashSet<>(baseSchema.getPrimaryKeys())
@@ -529,51 +537,53 @@ private static List<ChangeEvent> getPrimaryKeyChangeEvents(
           basePrimaryKeys.stream()
               .filter(key -> !targetPrimaryKeys.contains(key))
               .collect(Collectors.toSet());
+      for (String removedBaseKeyField : removedBaseKeys) {
+        Urn schemaFieldUrn = getSchemaFieldUrn(datasetUrn.toString(), removedBaseKeyField);
+        primaryKeyChangeEvents.add(
+            DatasetSchemaFieldChangeEvent.schemaFieldChangeEventBuilder()
+                .category(ChangeCategory.TECHNICAL_SCHEMA)
+                .modifier(schemaFieldUrn.toString())
+                .fieldUrn(schemaFieldUrn)
+                .fieldPath(removedBaseKeyField)
+                .entityUrn(datasetUrn.toString())
+                .operation(ChangeOperation.MODIFY)
+                .semVerChange(SemanticChangeType.MAJOR)
+                .description(
+                    BACKWARDS_INCOMPATIBLE_DESC
+                        + " removal of the primary key field '"
+                        + removedBaseKeyField
+                        + "'")
+                .auditStamp(auditStamp)
+                .modificationCategory(SchemaFieldModificationCategory.OTHER)
+                .build());
+      }
 
       Set<String> addedTargetKeys =
           targetPrimaryKeys.stream()
               .filter(key -> !basePrimaryKeys.contains(key))
               .collect(Collectors.toSet());
-      if (!removedBaseKeys.isEmpty() || !addedTargetKeys.isEmpty()) {
-        String keyChangeTarget;
-        // Just pick the first schema field we can find for the change event
-        if (!removedBaseKeys.isEmpty()) {
-          keyChangeTarget = removedBaseKeys.stream().findFirst().get();
-        } else {
-          keyChangeTarget = addedTargetKeys.stream().findFirst().get();
-        }
-
-        StringBuilder description =
-            new StringBuilder(BACKWARDS_INCOMPATIBLE_DESC + " a primary key constraint change.");
-        if (!removedBaseKeys.isEmpty()) {
-          description.append(" The following fields were removed:");
-          removedBaseKeys.forEach(
-              removedBaseKey -> description.append(" '").append(removedBaseKey).append("'"));
-          description.append(".");
-        }
-        if (!addedTargetKeys.isEmpty()) {
-          description.append(" The following fields were added:");
-          addedTargetKeys.forEach(
-              addedTargetKey -> description.append(" '").append(addedTargetKey).append("'"));
-          description.append(".");
-        }
+      for (String addedTargetKeyField : addedTargetKeys) {
+        Urn schemaFieldUrn = getSchemaFieldUrn(datasetUrn.toString(), addedTargetKeyField);
         primaryKeyChangeEvents.add(
             DatasetSchemaFieldChangeEvent.schemaFieldChangeEventBuilder()
                 .category(ChangeCategory.TECHNICAL_SCHEMA)
-                .fieldUrn(getSchemaFieldUrn(datasetUrn, keyChangeTarget))
-                .fieldPath(keyChangeTarget)
-                .modifier(getSchemaFieldUrn(datasetUrn, keyChangeTarget).toString())
+                .modifier(getSchemaFieldUrn(datasetUrn, addedTargetKeyField).toString())
+                .fieldUrn(schemaFieldUrn)
+                .fieldPath(addedTargetKeyField)
                 .entityUrn(datasetUrn.toString())
                 .operation(ChangeOperation.MODIFY)
                 .semVerChange(SemanticChangeType.MAJOR)
-                .description(description.toString())
-                .modificationCategory(SchemaFieldModificationCategory.OTHER)
+                .description(
+                    BACKWARDS_INCOMPATIBLE_DESC
+                        + " addition of the primary key field '"
+                        + addedTargetKeyField
+                        + "'")
                 .auditStamp(auditStamp)
+                .modificationCategory(SchemaFieldModificationCategory.OTHER)
                 .build());
-        return primaryKeyChangeEvents;
       }
     }
-    return Collections.emptyList();
+    return primaryKeyChangeEvents;
   }
 
   @Override
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java
index 67518121edae4..4d940c229dc9a 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java
@@ -13,6 +13,7 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.ByteString;
 import com.linkedin.metadata.aspect.EnvelopedAspect;
+import com.linkedin.metadata.config.TimeseriesAspectServiceConfig;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.models.annotation.SearchableAnnotation;
@@ -53,8 +54,15 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -103,18 +111,29 @@ public class ElasticSearchTimeseriesAspectService
   private final RestHighLevelClient searchClient;
   private final ESAggregatedStatsDAO esAggregatedStatsDAO;
   private final QueryFilterRewriteChain queryFilterRewriteChain;
+  private final ExecutorService queryPool;
 
   public ElasticSearchTimeseriesAspectService(
       @Nonnull RestHighLevelClient searchClient,
       @Nonnull TimeseriesAspectIndexBuilders indexBuilders,
       @Nonnull ESBulkProcessor bulkProcessor,
       int numRetries,
-      @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) {
+      @Nonnull QueryFilterRewriteChain queryFilterRewriteChain,
+      @Nonnull TimeseriesAspectServiceConfig timeseriesAspectServiceConfig) {
     this.indexBuilders = indexBuilders;
     this.searchClient = searchClient;
     this.bulkProcessor = bulkProcessor;
     this.numRetries = numRetries;
     this.queryFilterRewriteChain = queryFilterRewriteChain;
+    this.queryPool =
+        new ThreadPoolExecutor(
+            timeseriesAspectServiceConfig.getQuery().getConcurrency(), // core threads
+            timeseriesAspectServiceConfig.getQuery().getConcurrency(), // max threads
+            timeseriesAspectServiceConfig.getQuery().getKeepAlive(),
+            TimeUnit.SECONDS, // thread keep-alive time
+            new ArrayBlockingQueue<>(
+                timeseriesAspectServiceConfig.getQuery().getQueueSize()), // fixed size queue
+            new ThreadPoolExecutor.CallerRunsPolicy());
 
     esAggregatedStatsDAO = new ESAggregatedStatsDAO(searchClient, queryFilterRewriteChain);
   }
@@ -400,6 +419,69 @@ public List<EnvelopedAspect> getAspectValues(
         .collect(Collectors.toList());
   }
 
+  @Nonnull
+  @Override
+  public Map<Urn, Map<String, EnvelopedAspect>> getLatestTimeseriesAspectValues(
+      @Nonnull OperationContext opContext,
+      @Nonnull Set<Urn> urns,
+      @Nonnull Set<String> aspectNames,
+      @Nullable Map<String, Long> endTimeMillis) {
+    Map<Urn, List<Future<Pair<String, EnvelopedAspect>>>> futures =
+        urns.stream()
+            .map(
+                urn -> {
+                  List<Future<Pair<String, EnvelopedAspect>>> aspectFutures =
+                      aspectNames.stream()
+                          .map(
+                              aspectName ->
+                                  queryPool.submit(
+                                      () -> {
+                                        List<EnvelopedAspect> oneResultList =
+                                            getAspectValues(
+                                                opContext,
+                                                urn,
+                                                urn.getEntityType(),
+                                                aspectName,
+                                                null,
+                                                endTimeMillis == null
+                                                    ? null
+                                                    : endTimeMillis.get(aspectName),
+                                                1,
+                                                null,
+                                                null);
+                                        return !oneResultList.isEmpty()
+                                            ? Pair.of(aspectName, oneResultList.get(0))
+                                            : null;
+                                      }))
+                          .collect(Collectors.toList());
+
+                  return Map.entry(urn, aspectFutures);
+                })
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+    return futures.entrySet().stream()
+        .map(
+            e ->
+                Map.entry(
+                    e.getKey(),
+                    e.getValue().stream()
+                        .map(
+                            f -> {
+                              try {
+                                return f.get();
+                              } catch (InterruptedException | ExecutionException ex) {
+                                throw new RuntimeException(ex);
+                              }
+                            })
+                        .filter(Objects::nonNull)
+                        .collect(Collectors.toList())))
+        .collect(
+            Collectors.toMap(
+                Map.Entry::getKey,
+                e ->
+                    e.getValue().stream().collect(Collectors.toMap(Pair::getKey, Pair::getValue))));
+  }
+
   @Override
   @Nonnull
   public GenericTable getAggregatedStats(
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java
index 346a1eef84592..395c040f28811 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java
@@ -34,19 +34,19 @@ public static SystemMetadata createSystemMetadata() {
   }
 
   @Nonnull
-  public static SystemMetadata createSystemMetadata(long nextAspectVersion) {
+  public static SystemMetadata createSystemMetadata(int nextAspectVersion) {
     return createSystemMetadata(
         1625792689, "run-123", "run-123", String.valueOf(nextAspectVersion));
   }
 
   @Nonnull
-  public static SystemMetadata createSystemMetadata(long lastObserved, @Nonnull String runId) {
+  public static SystemMetadata createSystemMetadata(int lastObserved, @Nonnull String runId) {
     return createSystemMetadata(lastObserved, runId, runId, null);
   }
 
   @Nonnull
   public static SystemMetadata createSystemMetadata(
-      long lastObserved,
+      int lastObserved, // for test comparison must be int
       @Nonnull String runId,
       @Nonnull String lastRunId,
       @Nullable String version) {
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java
index 12b12cf105196..fa6ab7932001b 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java
@@ -46,12 +46,12 @@ public static Map<Urn, CorpUserKey> ingestCorpUserKeyAspects(
               .recordTemplate(aspect)
               .auditStamp(AspectGenerationUtils.createAuditStamp())
               .systemMetadata(AspectGenerationUtils.createSystemMetadata())
-              .build(opContext.getAspectRetrieverOpt().get()));
+              .build(opContext.getAspectRetriever()));
     }
     entityService.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -83,12 +83,12 @@ public static Map<Urn, CorpUserInfo> ingestCorpUserInfoAspects(
               .recordTemplate(aspect)
               .auditStamp(AspectGenerationUtils.createAuditStamp())
               .systemMetadata(AspectGenerationUtils.createSystemMetadata())
-              .build(opContext.getAspectRetrieverOpt().get()));
+              .build(opContext.getAspectRetriever()));
     }
     entityService.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -121,12 +121,12 @@ public static Map<Urn, ChartInfo> ingestChartInfoAspects(
               .recordTemplate(aspect)
               .auditStamp(AspectGenerationUtils.createAuditStamp())
               .systemMetadata(AspectGenerationUtils.createSystemMetadata())
-              .build(opContext.getAspectRetrieverOpt().get()));
+              .build(opContext.getAspectRetriever()));
     }
     entityService.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/aspect/hooks/IgnoreUnknownMutatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/aspect/hooks/IgnoreUnknownMutatorTest.java
index 11a3153abcaee..19be1eb14667d 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/aspect/hooks/IgnoreUnknownMutatorTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/aspect/hooks/IgnoreUnknownMutatorTest.java
@@ -16,7 +16,8 @@
 import com.linkedin.data.template.StringMap;
 import com.linkedin.dataset.DatasetProperties;
 import com.linkedin.events.metadata.ChangeType;
-import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
+import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
 import com.linkedin.metadata.entity.SearchRetriever;
@@ -28,7 +29,6 @@
 import com.linkedin.mxe.SystemMetadata;
 import com.linkedin.test.metadata.aspect.TestEntityRegistry;
 import io.datahubproject.metadata.context.RetrieverContext;
-import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
@@ -53,17 +53,17 @@ public class IgnoreUnknownMutatorTest {
   private static final Urn TEST_DATASET_URN =
       UrnUtils.getUrn(
           "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)");
-  private AspectRetriever mockAspectRetriever;
+  private CachingAspectRetriever mockAspectRetriever;
   private RetrieverContext retrieverContext;
 
   @BeforeMethod
   public void setup() {
-    mockAspectRetriever = mock(AspectRetriever.class);
+    mockAspectRetriever = mock(CachingAspectRetriever.class);
     retrieverContext =
         RetrieverContext.builder()
             .searchRetriever(mock(SearchRetriever.class))
-            .aspectRetriever(mockAspectRetriever)
-            .graphRetriever(TestOperationContexts.emptyGraphRetriever)
+            .cachingAspectRetriever(mockAspectRetriever)
+            .graphRetriever(GraphRetriever.EMPTY)
             .build();
   }
 
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtilTest.java b/metadata-io/src/test/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtilTest.java
index 04aff4edf456d..e7ed267113159 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtilTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtilTest.java
@@ -56,8 +56,7 @@ public void testAdditionalChanges() {
         DefaultAspectsUtil.getAdditionalChanges(
                 opContext,
                 AspectsBatchImpl.builder()
-                    .mcps(
-                        List.of(proposal1), new AuditStamp(), opContext.getRetrieverContext().get())
+                    .mcps(List.of(proposal1), new AuditStamp(), opContext.getRetrieverContext())
                     .build()
                     .getMCPItems(),
                 entityServiceImpl,
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidatorTest.java
new file mode 100644
index 0000000000000..f46772ca7b350
--- /dev/null
+++ b/metadata-io/src/test/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidatorTest.java
@@ -0,0 +1,166 @@
+package com.linkedin.metadata.aspect.validation;
+
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_ENTITY_NAME;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_ABORTED;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_CANCELLED;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_DUPLICATE;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_FAILURE;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_RUNNING;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_SUCCESS;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_TIMEOUT;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.execution.ExecutionRequestResult;
+import com.linkedin.metadata.aspect.RetrieverContext;
+import com.linkedin.metadata.aspect.SystemAspect;
+import com.linkedin.metadata.aspect.batch.ChangeMCP;
+import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
+import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
+import com.linkedin.metadata.utils.AuditStampUtils;
+import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.testng.annotations.Test;
+
+public class ExecutionRequestResultValidatorTest {
+  private static final OperationContext TEST_CONTEXT =
+      TestOperationContexts.systemContextNoSearchAuthorization();
+  private static final AspectPluginConfig TEST_PLUGIN_CONFIG =
+      AspectPluginConfig.builder()
+          .className(ExecutionRequestResultValidator.class.getName())
+          .enabled(true)
+          .supportedOperations(List.of("UPSERT"))
+          .supportedEntityAspectNames(
+              List.of(
+                  AspectPluginConfig.EntityAspectName.builder()
+                      .entityName(EXECUTION_REQUEST_ENTITY_NAME)
+                      .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME)
+                      .build()))
+          .build();
+  private static final Urn TEST_URN = UrnUtils.getUrn("urn:li:dataHubExecutionRequest:xyz");
+
+  @Test
+  public void testAllowed() {
+    ExecutionRequestResultValidator test = new ExecutionRequestResultValidator();
+    test.setConfig(TEST_PLUGIN_CONFIG);
+
+    Set<String> allowedUpdateStates =
+        Set.of(
+            EXECUTION_REQUEST_STATUS_RUNNING,
+            EXECUTION_REQUEST_STATUS_FAILURE,
+            EXECUTION_REQUEST_STATUS_TIMEOUT);
+    Set<String> destinationStates = new HashSet<>(allowedUpdateStates);
+    destinationStates.addAll(
+        Set.of(
+            EXECUTION_REQUEST_STATUS_ABORTED,
+            EXECUTION_REQUEST_STATUS_CANCELLED,
+            EXECUTION_REQUEST_STATUS_SUCCESS,
+            EXECUTION_REQUEST_STATUS_DUPLICATE));
+
+    List<ChangeMCP> testItems =
+        new ArrayList<>(
+            // Tests with previous state
+            allowedUpdateStates.stream()
+                .flatMap(
+                    prevState ->
+                        destinationStates.stream()
+                            .map(
+                                destState -> {
+                                  SystemAspect prevData = mock(SystemAspect.class);
+                                  when(prevData.getRecordTemplate())
+                                      .thenReturn(
+                                          new ExecutionRequestResult().setStatus(prevState));
+                                  return ChangeItemImpl.builder()
+                                      .changeType(ChangeType.UPSERT)
+                                      .urn(TEST_URN)
+                                      .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME)
+                                      .recordTemplate(
+                                          new ExecutionRequestResult().setStatus(destState))
+                                      .previousSystemAspect(prevData)
+                                      .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+                                      .build(TEST_CONTEXT.getAspectRetriever());
+                                }))
+                .toList());
+    // Tests with no previous
+    testItems.addAll(
+        destinationStates.stream()
+            .map(
+                destState ->
+                    ChangeItemImpl.builder()
+                        .changeType(ChangeType.UPSERT)
+                        .urn(TEST_URN)
+                        .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME)
+                        .recordTemplate(new ExecutionRequestResult().setStatus(destState))
+                        .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+                        .build(TEST_CONTEXT.getAspectRetriever()))
+            .toList());
+
+    List<AspectValidationException> result =
+        test.validatePreCommitAspects(testItems, mock(RetrieverContext.class)).toList();
+
+    assertTrue(result.isEmpty(), "Did not expect any validation errors.");
+  }
+
+  @Test
+  public void testDenied() {
+    ExecutionRequestResultValidator test = new ExecutionRequestResultValidator();
+    test.setConfig(TEST_PLUGIN_CONFIG);
+
+    Set<String> deniedUpdateStates =
+        Set.of(
+            EXECUTION_REQUEST_STATUS_ABORTED,
+            EXECUTION_REQUEST_STATUS_CANCELLED,
+            EXECUTION_REQUEST_STATUS_SUCCESS,
+            EXECUTION_REQUEST_STATUS_DUPLICATE);
+    Set<String> destinationStates = new HashSet<>(deniedUpdateStates);
+    destinationStates.addAll(
+        Set.of(
+            EXECUTION_REQUEST_STATUS_RUNNING,
+            EXECUTION_REQUEST_STATUS_FAILURE,
+            EXECUTION_REQUEST_STATUS_TIMEOUT));
+
+    List<ChangeMCP> testItems =
+        new ArrayList<>(
+            // Tests with previous state
+            deniedUpdateStates.stream()
+                .flatMap(
+                    prevState ->
+                        destinationStates.stream()
+                            .map(
+                                destState -> {
+                                  SystemAspect prevData = mock(SystemAspect.class);
+                                  when(prevData.getRecordTemplate())
+                                      .thenReturn(
+                                          new ExecutionRequestResult().setStatus(prevState));
+                                  return ChangeItemImpl.builder()
+                                      .changeType(ChangeType.UPSERT)
+                                      .urn(TEST_URN)
+                                      .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME)
+                                      .recordTemplate(
+                                          new ExecutionRequestResult().setStatus(destState))
+                                      .previousSystemAspect(prevData)
+                                      .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+                                      .build(TEST_CONTEXT.getAspectRetriever());
+                                }))
+                .toList());
+
+    List<AspectValidationException> result =
+        test.validatePreCommitAspects(testItems, mock(RetrieverContext.class)).toList();
+
+    assertEquals(
+        result.size(),
+        deniedUpdateStates.size() * destinationStates.size(),
+        "Expected ALL items to be denied.");
+  }
+}
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java b/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java
index 2ca966b104e03..4d977d179f91e 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java
@@ -5,9 +5,23 @@
 import static org.testng.Assert.assertThrows;
 
 import com.codahale.metrics.Counter;
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.Status;
+import com.linkedin.common.UrnArray;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.RequiredFieldNotPresentException;
+import com.linkedin.domain.Domains;
+import com.linkedin.entity.client.EntityClientConfig;
+import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.aspect.batch.AspectsBatch;
 import com.linkedin.metadata.entity.DeleteEntityService;
 import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.entity.IngestResult;
+import com.linkedin.metadata.entity.UpdateAspectResult;
+import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
+import com.linkedin.metadata.entity.ebean.batch.ProposedItem;
 import com.linkedin.metadata.event.EventProducer;
 import com.linkedin.metadata.search.EntitySearchService;
 import com.linkedin.metadata.search.LineageSearchService;
@@ -15,8 +29,14 @@
 import com.linkedin.metadata.search.client.CachingEntitySearchService;
 import com.linkedin.metadata.service.RollbackService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
+import com.linkedin.metadata.utils.AuditStampUtils;
+import com.linkedin.metadata.utils.GenericRecordUtils;
 import com.linkedin.metadata.utils.metrics.MetricUtils;
+import com.linkedin.mxe.MetadataChangeProposal;
+import com.linkedin.r2.RemoteInvocationException;
 import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
+import java.util.List;
 import java.util.function.Supplier;
 import org.mockito.MockedStatic;
 import org.testng.annotations.AfterMethod;
@@ -25,7 +45,7 @@
 
 public class JavaEntityClientTest {
 
-  private EntityService _entityService;
+  private EntityService<?> _entityService;
   private DeleteEntityService _deleteEntityService;
   private EntitySearchService _entitySearchService;
   private CachingEntitySearchService _cachingEntitySearchService;
@@ -52,7 +72,7 @@ public void setupTest() {
     _metricUtils = mockStatic(MetricUtils.class);
     _counter = mock(Counter.class);
     when(MetricUtils.counter(any(), any())).thenReturn(_counter);
-    opContext = mock(OperationContext.class);
+    opContext = TestOperationContexts.systemContextNoSearchAuthorization();
   }
 
   @AfterMethod
@@ -71,7 +91,7 @@ private JavaEntityClient getJavaEntityClient() {
         _timeseriesAspectService,
         rollbackService,
         _eventProducer,
-        1);
+        EntityClientConfig.builder().batchGetV2Size(1).build());
   }
 
   @Test
@@ -131,4 +151,97 @@ void testThrowAfterNonRetryableException() {
         () -> MetricUtils.counter(client.getClass(), "exception_" + e.getClass().getName()),
         times(1));
   }
+
+  @Test
+  void tesIngestOrderingWithProposedItem() throws RemoteInvocationException {
+    JavaEntityClient client = getJavaEntityClient();
+    Urn testUrn = UrnUtils.getUrn("urn:li:container:orderingTest");
+    AuditStamp auditStamp = AuditStampUtils.createDefaultAuditStamp();
+    MetadataChangeProposal mcp =
+        new MetadataChangeProposal()
+            .setEntityUrn(testUrn)
+            .setAspectName("status")
+            .setEntityType("container")
+            .setChangeType(ChangeType.UPSERT)
+            .setAspect(GenericRecordUtils.serializeAspect(new Status().setRemoved(true)));
+
+    when(_entityService.ingestProposal(
+            any(OperationContext.class), any(AspectsBatch.class), eq(false)))
+        .thenReturn(
+            List.<IngestResult>of(
+                // Misc - unrelated urn
+                IngestResult.builder()
+                    .urn(UrnUtils.getUrn("urn:li:container:domains"))
+                    .request(
+                        ChangeItemImpl.builder()
+                            .entitySpec(
+                                opContext
+                                    .getEntityRegistry()
+                                    .getEntitySpec(Constants.CONTAINER_ENTITY_NAME))
+                            .aspectSpec(
+                                opContext
+                                    .getEntityRegistry()
+                                    .getEntitySpec(Constants.CONTAINER_ENTITY_NAME)
+                                    .getAspectSpec(Constants.DOMAINS_ASPECT_NAME))
+                            .changeType(ChangeType.UPSERT)
+                            .urn(UrnUtils.getUrn("urn:li:container:domains"))
+                            .aspectName("domains")
+                            .recordTemplate(new Domains().setDomains(new UrnArray()))
+                            .auditStamp(auditStamp)
+                            .build(opContext.getAspectRetriever()))
+                    .isUpdate(true)
+                    .publishedMCL(true)
+                    .sqlCommitted(true)
+                    .build(),
+                // Side effect - unrelated urn
+                IngestResult.builder()
+                    .urn(UrnUtils.getUrn("urn:li:container:sideEffect"))
+                    .request(
+                        ChangeItemImpl.builder()
+                            .entitySpec(
+                                opContext
+                                    .getEntityRegistry()
+                                    .getEntitySpec(Constants.CONTAINER_ENTITY_NAME))
+                            .aspectSpec(
+                                opContext
+                                    .getEntityRegistry()
+                                    .getEntitySpec(Constants.CONTAINER_ENTITY_NAME)
+                                    .getAspectSpec(Constants.STATUS_ASPECT_NAME))
+                            .changeType(ChangeType.UPSERT)
+                            .urn(UrnUtils.getUrn("urn:li:container:sideEffect"))
+                            .aspectName("status")
+                            .recordTemplate(new Status().setRemoved(false))
+                            .auditStamp(auditStamp)
+                            .build(opContext.getAspectRetriever()))
+                    .isUpdate(true)
+                    .publishedMCL(true)
+                    .sqlCommitted(true)
+                    .build(),
+                // Expected response
+                IngestResult.builder()
+                    .urn(testUrn)
+                    .request(
+                        ProposedItem.builder()
+                            .metadataChangeProposal(mcp)
+                            .entitySpec(
+                                opContext
+                                    .getEntityRegistry()
+                                    .getEntitySpec(Constants.CONTAINER_ENTITY_NAME))
+                            .aspectSpec(
+                                opContext
+                                    .getEntityRegistry()
+                                    .getEntitySpec(Constants.CONTAINER_ENTITY_NAME)
+                                    .getAspectSpec(Constants.STATUS_ASPECT_NAME))
+                            .auditStamp(auditStamp)
+                            .build())
+                    .result(UpdateAspectResult.builder().mcp(mcp).urn(testUrn).build())
+                    .isUpdate(true)
+                    .publishedMCL(true)
+                    .sqlCommitted(true)
+                    .build()));
+
+    String urnStr = client.ingestProposal(opContext, mcp, false);
+
+    assertEquals(urnStr, "urn:li:container:orderingTest");
+  }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java b/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java
index 1151014bf1162..215e1e2431efa 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java
@@ -15,8 +15,9 @@
 import com.linkedin.dataproduct.DataProductAssociationArray;
 import com.linkedin.dataproduct.DataProductProperties;
 import com.linkedin.events.metadata.ChangeType;
-import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.aspect.GraphRetriever;
+import com.linkedin.metadata.aspect.SystemAspect;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.models.graph.RelatedEntities;
 import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult;
@@ -34,6 +35,8 @@
 import com.linkedin.metadata.utils.AuditStampUtils;
 import com.linkedin.test.metadata.aspect.TestEntityRegistry;
 import io.datahubproject.metadata.context.RetrieverContext;
+import jakarta.json.JsonArray;
+import jakarta.json.JsonObject;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -45,13 +48,7 @@
 public class DataProductUnsetSideEffectTest {
   private static final EntityRegistry TEST_REGISTRY = new TestEntityRegistry();
   private static final List<ChangeType> SUPPORTED_CHANGE_TYPES =
-      List.of(
-          ChangeType.CREATE,
-          ChangeType.PATCH,
-          ChangeType.CREATE_ENTITY,
-          ChangeType.UPSERT,
-          ChangeType.DELETE,
-          ChangeType.RESTATE);
+      List.of(ChangeType.CREATE, ChangeType.CREATE_ENTITY, ChangeType.UPSERT, ChangeType.RESTATE);
   private static final Urn TEST_PRODUCT_URN =
       UrnUtils.getUrn("urn:li:dataProduct:someDataProductId");
 
@@ -78,12 +75,12 @@ public class DataProductUnsetSideEffectTest {
                       .build()))
           .build();
 
-  private AspectRetriever mockAspectRetriever;
+  private CachingAspectRetriever mockAspectRetriever;
   private RetrieverContext retrieverContext;
 
   @BeforeMethod
   public void setup() {
-    mockAspectRetriever = mock(AspectRetriever.class);
+    mockAspectRetriever = mock(CachingAspectRetriever.class);
     when(mockAspectRetriever.getEntityRegistry()).thenReturn(TEST_REGISTRY);
     GraphRetriever graphRetriever = mock(GraphRetriever.class);
     RelatedEntities relatedEntities =
@@ -142,7 +139,7 @@ public void setup() {
     retrieverContext =
         RetrieverContext.builder()
             .searchRetriever(mock(SearchRetriever.class))
-            .aspectRetriever(mockAspectRetriever)
+            .cachingAspectRetriever(mockAspectRetriever)
             .graphRetriever(graphRetriever)
             .build();
   }
@@ -251,6 +248,214 @@ public void testDPRemoveOld() {
                 .build(mockAspectRetriever.getEntityRegistry())));
   }
 
+  @Test
+  public void testBulkAssetMove() {
+    DataProductUnsetSideEffect test = new DataProductUnsetSideEffect();
+    test.setConfig(TEST_PLUGIN_CONFIG);
+
+    // Create 100 dataset URNs and set up their existing relationships
+    List<Urn> datasetUrns = new ArrayList<>();
+    for (int i = 0; i < 100; i++) {
+      Urn datasetUrn =
+          UrnUtils.getUrn(
+              String.format("urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_%d,PROD)", i));
+      datasetUrns.add(datasetUrn);
+
+      // Mock the existing relationship for each dataset with the old data product
+      RelatedEntities relatedEntities =
+          new RelatedEntities(
+              "DataProductContains",
+              TEST_PRODUCT_URN_2.toString(), // Old data product
+              datasetUrn.toString(),
+              RelationshipDirection.INCOMING,
+              null);
+
+      List<RelatedEntities> relatedEntitiesList = new ArrayList<>();
+      relatedEntitiesList.add(relatedEntities);
+      RelatedEntitiesScrollResult relatedEntitiesScrollResult =
+          new RelatedEntitiesScrollResult(1, 10, null, relatedEntitiesList);
+
+      when(retrieverContext
+              .getGraphRetriever()
+              .scrollRelatedEntities(
+                  eq(null),
+                  eq(QueryUtils.newFilter("urn", datasetUrn.toString())),
+                  eq(null),
+                  eq(EMPTY_FILTER),
+                  eq(ImmutableList.of("DataProductContains")),
+                  eq(
+                      QueryUtils.newRelationshipFilter(
+                          EMPTY_FILTER, RelationshipDirection.INCOMING)),
+                  eq(Collections.emptyList()),
+                  eq(null),
+                  eq(10),
+                  eq(null),
+                  eq(null)))
+          .thenReturn(relatedEntitiesScrollResult);
+    }
+
+    // Create data product properties with all 100 assets
+    DataProductProperties dataProductProperties = new DataProductProperties();
+    DataProductAssociationArray dataProductAssociations = new DataProductAssociationArray();
+    for (Urn datasetUrn : datasetUrns) {
+      DataProductAssociation association = new DataProductAssociation();
+      association.setDestinationUrn(datasetUrn);
+      dataProductAssociations.add(association);
+    }
+    dataProductProperties.setAssets(dataProductAssociations);
+
+    // Run test
+    ChangeItemImpl dataProductPropertiesChangeItem =
+        ChangeItemImpl.builder()
+            .urn(TEST_PRODUCT_URN) // New data product
+            .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)
+            .changeType(ChangeType.UPSERT)
+            .entitySpec(TEST_REGISTRY.getEntitySpec(DATA_PRODUCT_ENTITY_NAME))
+            .aspectSpec(
+                TEST_REGISTRY
+                    .getEntitySpec(DATA_PRODUCT_ENTITY_NAME)
+                    .getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME))
+            .recordTemplate(dataProductProperties)
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(mockAspectRetriever);
+
+    List<MCPItem> testOutput =
+        test.postMCPSideEffect(
+                List.of(
+                    MCLItemImpl.builder()
+                        .build(
+                            dataProductPropertiesChangeItem,
+                            null,
+                            null,
+                            retrieverContext.getAspectRetriever())),
+                retrieverContext)
+            .toList();
+
+    // Verify test
+    assertEquals(testOutput.size(), 1, "Expected one patch to remove assets from old data product");
+
+    MCPItem patchItem = testOutput.get(0);
+    assertEquals(
+        patchItem.getUrn(), TEST_PRODUCT_URN_2, "Patch should target the old data product");
+    assertEquals(patchItem.getAspectName(), DATA_PRODUCT_PROPERTIES_ASPECT_NAME);
+
+    // Verify the patch contains remove operations for all 100 assets
+    JsonArray patchArray = ((PatchItemImpl) patchItem).getPatch().toJsonArray();
+    assertEquals(patchArray.size(), 100, "Should have 100 remove operations");
+
+    // Verify each remove operation
+    for (int i = 0; i < 100; i++) {
+      JsonObject op = patchArray.getJsonObject(i);
+      assertEquals(op.getString("op"), PatchOperationType.REMOVE.getValue());
+      assertEquals(
+          op.getString("path"),
+          String.format("/assets/urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_%d,PROD)", i));
+    }
+  }
+
+  @Test
+  public void testUpsertWithPreviousAspect() {
+    DataProductUnsetSideEffect test = new DataProductUnsetSideEffect();
+    test.setConfig(TEST_PLUGIN_CONFIG);
+
+    // Case 1: UPSERT with new additions
+    DataProductProperties previousProperties = new DataProductProperties();
+    DataProductAssociationArray previousAssociations = new DataProductAssociationArray();
+    DataProductAssociation previousAssociation = new DataProductAssociation();
+    previousAssociation.setDestinationUrn(DATASET_URN_1);
+    previousAssociations.add(previousAssociation);
+    previousProperties.setAssets(previousAssociations);
+
+    // New properties include both old and new datasets
+    DataProductProperties newProperties = new DataProductProperties();
+    DataProductAssociationArray newAssociations = new DataProductAssociationArray();
+    DataProductAssociation association1 = new DataProductAssociation();
+    association1.setDestinationUrn(DATASET_URN_1);
+    DataProductAssociation association2 = new DataProductAssociation();
+    association2.setDestinationUrn(DATASET_URN_2);
+    newAssociations.add(association1);
+    newAssociations.add(association2);
+    newProperties.setAssets(newAssociations);
+
+    // Create change item with previous aspect
+    SystemAspect prevData = mock(SystemAspect.class);
+    when(prevData.getRecordTemplate()).thenReturn(previousProperties);
+
+    ChangeItemImpl dataProductPropertiesChangeItem =
+        ChangeItemImpl.builder()
+            .urn(TEST_PRODUCT_URN)
+            .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)
+            .changeType(ChangeType.UPSERT)
+            .entitySpec(TEST_REGISTRY.getEntitySpec(DATA_PRODUCT_ENTITY_NAME))
+            .aspectSpec(
+                TEST_REGISTRY
+                    .getEntitySpec(DATA_PRODUCT_ENTITY_NAME)
+                    .getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME))
+            .recordTemplate(newProperties)
+            .previousSystemAspect(prevData)
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(mockAspectRetriever);
+
+    List<MCPItem> testOutput =
+        test.postMCPSideEffect(
+                List.of(
+                    MCLItemImpl.builder()
+                        .build(
+                            dataProductPropertiesChangeItem,
+                            null,
+                            null,
+                            retrieverContext.getAspectRetriever())),
+                retrieverContext)
+            .toList();
+
+    // Verify that only one patch is generated for the new dataset
+    assertEquals(
+        testOutput.size(), 1, "Expected removal of previous data product for new dataset only");
+    MCPItem patchItem = testOutput.get(0);
+    assertEquals(
+        patchItem.getUrn(), TEST_PRODUCT_URN_2, "Patch should target the old data product");
+    GenericJsonPatch.PatchOp expectedPatchOp = new GenericJsonPatch.PatchOp();
+    expectedPatchOp.setOp(PatchOperationType.REMOVE.getValue());
+    expectedPatchOp.setPath(String.format("/assets/%s", DATASET_URN_2));
+
+    // Case 2: UPSERT with no new additions
+    DataProductProperties sameProperties = new DataProductProperties();
+    DataProductAssociationArray sameAssociations = new DataProductAssociationArray();
+    DataProductAssociation sameAssociation = new DataProductAssociation();
+    sameAssociation.setDestinationUrn(DATASET_URN_1);
+    sameAssociations.add(sameAssociation);
+    sameProperties.setAssets(sameAssociations);
+
+    SystemAspect prevSameData = mock(SystemAspect.class);
+    when(prevData.getRecordTemplate()).thenReturn(sameProperties);
+
+    ChangeItemImpl noChangeItem =
+        ChangeItemImpl.builder()
+            .urn(TEST_PRODUCT_URN)
+            .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)
+            .changeType(ChangeType.UPSERT)
+            .entitySpec(TEST_REGISTRY.getEntitySpec(DATA_PRODUCT_ENTITY_NAME))
+            .aspectSpec(
+                TEST_REGISTRY
+                    .getEntitySpec(DATA_PRODUCT_ENTITY_NAME)
+                    .getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME))
+            .recordTemplate(sameProperties)
+            .previousSystemAspect(prevSameData)
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(mockAspectRetriever);
+
+    List<MCPItem> noChangeOutput =
+        test.postMCPSideEffect(
+                List.of(
+                    MCLItemImpl.builder()
+                        .build(noChangeItem, null, null, retrieverContext.getAspectRetriever())),
+                retrieverContext)
+            .toList();
+
+    // Verify no patches are generated when there are no new additions
+    assertEquals(noChangeOutput.size(), 0, "Expected no changes when assets are the same");
+  }
+
   private static DataProductProperties getTestDataProductProperties(Urn destinationUrn) {
     DataProductProperties dataProductProperties = new DataProductProperties();
     DataProductAssociationArray dataProductAssociations = new DataProductAssociationArray();
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java
index 0e8ee08e60739..723cb7813769f 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java
@@ -113,7 +113,7 @@ public void testDeleteUniqueRefGeneratesValidMCP() {
     dbValue.setCreatedOn(new Timestamp(auditStamp.getTime()));
 
     final Map<EntityAspectIdentifier, EntityAspect> dbEntries = Map.of(dbKey, dbValue);
-    Mockito.when(_aspectDao.batchGet(Mockito.any())).thenReturn(dbEntries);
+    Mockito.when(_aspectDao.batchGet(Mockito.any(), Mockito.anyBoolean())).thenReturn(dbEntries);
 
     RollbackResult result =
         new RollbackResult(
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityUtilsTest.java
index 943ad2967de42..f210ae94d0d70 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityUtilsTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityUtilsTest.java
@@ -1,14 +1,31 @@
 package com.linkedin.metadata.entity;
 
 import com.datahub.util.RecordUtils;
+import com.google.common.collect.ImmutableList;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.DataList;
 import com.linkedin.data.DataMap;
 import com.linkedin.data.schema.DataSchema;
 import com.linkedin.data.schema.PathSpec;
 import com.linkedin.data.schema.grammar.PdlSchemaParser;
 import com.linkedin.data.schema.resolver.DefaultDataSchemaResolver;
+import com.linkedin.data.template.StringArray;
 import com.linkedin.entity.Aspect;
+import com.linkedin.form.FormInfo;
+import com.linkedin.form.FormPrompt;
+import com.linkedin.form.FormPromptArray;
+import com.linkedin.form.StructuredPropertyParams;
+import com.linkedin.metadata.query.filter.Condition;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.Criterion;
+import com.linkedin.metadata.query.filter.CriterionArray;
+import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.schema.SchemaMetadata;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
 import junit.framework.TestCase;
 import org.testng.annotations.Test;
 
@@ -359,4 +376,60 @@ public void testSchemaMetadataDelete() {
                     .get("tags"))
             .size());
   }
+
+  @Test
+  public void testRemovePromptsFromFormInfo() {
+    Urn deletedPropertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:1");
+    Urn existingPropertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:2");
+    List<FormPrompt> prompts = new ArrayList<>();
+    prompts.add(
+        new FormPrompt()
+            .setId("1")
+            .setStructuredPropertyParams(
+                new StructuredPropertyParams().setUrn(deletedPropertyUrn)));
+    prompts.add(
+        new FormPrompt()
+            .setId("2")
+            .setStructuredPropertyParams(
+                new StructuredPropertyParams().setUrn(existingPropertyUrn)));
+    FormInfo formInfo = new FormInfo().setPrompts(new FormPromptArray(prompts));
+
+    FormInfo updatedFormInfo =
+        DeleteEntityUtils.removePromptsFromFormInfoAspect(formInfo, deletedPropertyUrn);
+
+    assertEquals(updatedFormInfo.getPrompts().size(), 1);
+    assertEquals(
+        updatedFormInfo.getPrompts(),
+        formInfo.getPrompts().stream()
+            .filter(prompt -> !prompt.getId().equals("1"))
+            .collect(Collectors.toList()));
+  }
+
+  @Test
+  public void testFilterForStructuredPropDeletion() {
+    Urn deletedPropertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:1");
+
+    final CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(
+        new Criterion()
+            .setField("structuredPropertyPromptUrns")
+            .setValues(new StringArray(deletedPropertyUrn.toString()))
+            .setNegated(false)
+            .setValue("")
+            .setCondition(Condition.EQUAL));
+    Filter expectedFilter =
+        new Filter()
+            .setOr(
+                new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(criterionArray)));
+
+    assertEquals(
+        DeleteEntityUtils.getFilterForStructuredPropertyDeletion(deletedPropertyUrn),
+        expectedFilter);
+  }
+
+  @Test
+  public void testEntityNamesForStructuredPropDeletion() {
+    assertEquals(
+        DeleteEntityUtils.getEntityNamesForStructuredPropertyDeletion(), ImmutableList.of("form"));
+  }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java
index e8d3c654f6f63..88f84ee94c8ee 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java
@@ -2,6 +2,7 @@
 
 import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME;
+import static com.linkedin.metadata.entity.ebean.EbeanAspectDao.TX_ISOLATION;
 import static org.mockito.Mockito.mock;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
@@ -18,6 +19,7 @@
 import com.linkedin.metadata.AspectGenerationUtils;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.EbeanTestUtils;
+import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.config.EbeanConfiguration;
 import com.linkedin.metadata.config.PreProcessHooks;
 import com.linkedin.metadata.entity.ebean.EbeanAspectDao;
@@ -39,7 +41,6 @@
 import io.ebean.Database;
 import io.ebean.Transaction;
 import io.ebean.TxScope;
-import io.ebean.annotation.TxIsolation;
 import java.net.URISyntaxException;
 import java.sql.Timestamp;
 import java.time.Instant;
@@ -98,13 +99,17 @@ public void setupTest() {
                             .entityService(_entityServiceImpl)
                             .entityRegistry(_testEntityRegistry)
                             .build())
-                    .graphRetriever(TestOperationContexts.emptyGraphRetriever)
-                    .searchRetriever(TestOperationContexts.emptySearchRetriever)
+                    .cachingAspectRetriever(
+                        TestOperationContexts.emptyActiveUsersAspectRetriever(
+                            () -> _testEntityRegistry))
+                    .graphRetriever(GraphRetriever.EMPTY)
+                    .searchRetriever(SearchRetriever.EMPTY)
                     .build(),
             null,
             opContext ->
-                ((EntityServiceAspectRetriever) opContext.getAspectRetrieverOpt().get())
-                    .setSystemOperationContext(opContext));
+                ((EntityServiceAspectRetriever) opContext.getAspectRetriever())
+                    .setSystemOperationContext(opContext),
+            null);
   }
 
   /**
@@ -151,25 +156,25 @@ public void testIngestListLatestAspects() throws AssertionError {
                 .recordTemplate(writeAspect1)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(TestOperationContexts.emptyAspectRetriever(null)),
+                .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null)),
             ChangeItemImpl.builder()
                 .urn(entityUrn2)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect2)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(TestOperationContexts.emptyAspectRetriever(null)),
+                .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null)),
             ChangeItemImpl.builder()
                 .urn(entityUrn3)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect3)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(TestOperationContexts.emptyAspectRetriever(null)));
+                .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null)));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -229,25 +234,25 @@ public void testIngestListUrns() throws AssertionError {
                 .recordTemplate(writeAspect1)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(TestOperationContexts.emptyAspectRetriever(null)),
+                .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null)),
             ChangeItemImpl.builder()
                 .urn(entityUrn2)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect2)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(TestOperationContexts.emptyAspectRetriever(null)),
+                .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null)),
             ChangeItemImpl.builder()
                 .urn(entityUrn3)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect3)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(TestOperationContexts.emptyAspectRetriever(null)));
+                .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null)));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -280,12 +285,11 @@ public void testNestedTransactions() throws AssertionError {
     Database server = _aspectDao.getServer();
 
     try (Transaction transaction =
-        server.beginTransaction(TxScope.requiresNew().setIsolation(TxIsolation.REPEATABLE_READ))) {
+        server.beginTransaction(TxScope.requiresNew().setIsolation(TX_ISOLATION))) {
       transaction.setBatchMode(true);
       // Work 1
       try (Transaction transaction2 =
-          server.beginTransaction(
-              TxScope.requiresNew().setIsolation(TxIsolation.REPEATABLE_READ))) {
+          server.beginTransaction(TxScope.requiresNew().setIsolation(TX_ISOLATION))) {
         transaction2.setBatchMode(true);
         // Work 2
         transaction2.commit();
@@ -310,11 +314,11 @@ public void testSystemMetadataDuplicateKey() throws Exception {
             .recordTemplate(new Status().setRemoved(true))
             .systemMetadata(systemMetadata)
             .auditStamp(TEST_AUDIT_STAMP)
-            .build(TestOperationContexts.emptyAspectRetriever(null));
+            .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(List.of(item))
             .build(),
         false,
@@ -336,7 +340,7 @@ public void testSystemMetadataDuplicateKey() throws Exception {
     try (Transaction transaction =
         ((EbeanAspectDao) _entityServiceImpl.aspectDao)
             .getServer()
-            .beginTransaction(TxScope.requiresNew().setIsolation(TxIsolation.REPEATABLE_READ))) {
+            .beginTransaction(TxScope.requiresNew().setIsolation(TX_ISOLATION))) {
       TransactionContext transactionContext = TransactionContext.empty(transaction, 3);
       _entityServiceImpl.aspectDao.saveAspect(
           transactionContext,
@@ -356,7 +360,7 @@ public void testSystemMetadataDuplicateKey() throws Exception {
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(
                 List.of(
                     ChangeItemImpl.builder()
@@ -365,7 +369,7 @@ public void testSystemMetadataDuplicateKey() throws Exception {
                         .recordTemplate(new Status().setRemoved(false))
                         .systemMetadata(systemMetadata)
                         .auditStamp(TEST_AUDIT_STAMP)
-                        .build(TestOperationContexts.emptyAspectRetriever(null))))
+                        .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null))))
             .build(),
         false,
         true);
@@ -381,57 +385,6 @@ public void testSystemMetadataDuplicateKey() throws Exception {
         "Expected version 0 with systemMeta version 3 accounting for the the collision");
   }
 
-  @Test
-  public void testBatchDuplicate() throws Exception {
-    Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:batchDuplicateTest");
-    SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata();
-    ChangeItemImpl item1 =
-        ChangeItemImpl.builder()
-            .urn(entityUrn)
-            .aspectName(STATUS_ASPECT_NAME)
-            .recordTemplate(new Status().setRemoved(true))
-            .systemMetadata(systemMetadata.copy())
-            .auditStamp(TEST_AUDIT_STAMP)
-            .build(TestOperationContexts.emptyAspectRetriever(null));
-    ChangeItemImpl item2 =
-        ChangeItemImpl.builder()
-            .urn(entityUrn)
-            .aspectName(STATUS_ASPECT_NAME)
-            .recordTemplate(new Status().setRemoved(false))
-            .systemMetadata(systemMetadata.copy())
-            .auditStamp(TEST_AUDIT_STAMP)
-            .build(TestOperationContexts.emptyAspectRetriever(null));
-    _entityServiceImpl.ingestAspects(
-        opContext,
-        AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
-            .items(List.of(item1, item2))
-            .build(),
-        false,
-        true);
-
-    // List aspects urns
-    ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 2);
-
-    assertEquals(batch.getStart().intValue(), 0);
-    assertEquals(batch.getCount().intValue(), 1);
-    assertEquals(batch.getTotal().intValue(), 1);
-    assertEquals(batch.getEntities().size(), 1);
-    assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString());
-
-    EnvelopedAspect envelopedAspect =
-        _entityServiceImpl.getLatestEnvelopedAspect(
-            opContext, CORP_USER_ENTITY_NAME, entityUrn, STATUS_ASPECT_NAME);
-    assertEquals(
-        envelopedAspect.getSystemMetadata().getVersion(),
-        "2",
-        "Expected version 2 accounting for duplicates");
-    assertEquals(
-        envelopedAspect.getValue().toString(),
-        "{removed=false}",
-        "Expected 2nd item to be the latest");
-  }
-
   @Test
   public void dataGeneratorThreadingTest() {
     DataGenerator dataGenerator = new DataGenerator(opContext, _entityServiceImpl);
@@ -467,7 +420,7 @@ public void multiThreadingTest() {
     List<List<MetadataChangeProposal>> testData =
         dataGenerator.generateMCPs("dataset", 25, aspects).collect(Collectors.toList());
 
-    executeThreadingTest(opContext, _entityServiceImpl, testData, 15);
+    executeThreadingTest(userContext, _entityServiceImpl, testData, 15);
 
     // Expected aspects
     Set<Triple<String, String, Long>> generatedAspectIds =
@@ -506,7 +459,9 @@ public void multiThreadingTest() {
     assertEquals(
         missing.size(),
         0,
-        String.format("Expected all generated aspects to be inserted. Missing: %s", missing));
+        String.format(
+            "Expected all generated aspects to be inserted. Missing Examples: %s",
+            missing.stream().limit(10).collect(Collectors.toSet())));
   }
 
   /**
@@ -523,7 +478,7 @@ public void singleThreadingTest() {
     List<List<MetadataChangeProposal>> testData =
         dataGenerator.generateMCPs("dataset", 25, aspects).collect(Collectors.toList());
 
-    executeThreadingTest(opContext, _entityServiceImpl, testData, 1);
+    executeThreadingTest(userContext, _entityServiceImpl, testData, 1);
 
     // Expected aspects
     Set<Triple<String, String, Long>> generatedAspectIds =
@@ -649,7 +604,7 @@ public void run() {
           auditStamp.setTime(System.currentTimeMillis());
           AspectsBatchImpl batch =
               AspectsBatchImpl.builder()
-                  .mcps(mcps, auditStamp, operationContext.getRetrieverContext().get())
+                  .mcps(mcps, auditStamp, operationContext.getRetrieverContext())
                   .build();
           entityService.ingestProposal(operationContext, batch, false);
         }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java
index 53f5ebfe59728..c00632e5cf542 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java
@@ -11,14 +11,18 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.AuditStamp;
+import com.linkedin.common.GlobalTags;
 import com.linkedin.common.Owner;
 import com.linkedin.common.OwnerArray;
 import com.linkedin.common.Ownership;
 import com.linkedin.common.OwnershipType;
 import com.linkedin.common.Status;
+import com.linkedin.common.TagAssociation;
+import com.linkedin.common.TagAssociationArray;
 import com.linkedin.common.UrnArray;
 import com.linkedin.common.VersionedUrn;
 import com.linkedin.common.urn.CorpuserUrn;
+import com.linkedin.common.urn.TagUrn;
 import com.linkedin.common.urn.TupleKey;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
@@ -42,22 +46,25 @@
 import com.linkedin.metadata.aspect.CorpUserAspect;
 import com.linkedin.metadata.aspect.CorpUserAspectArray;
 import com.linkedin.metadata.aspect.VersionedAspect;
+import com.linkedin.metadata.aspect.patch.GenericJsonPatch;
+import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
 import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
+import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl;
 import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs;
 import com.linkedin.metadata.entity.validation.ValidationApiUtils;
 import com.linkedin.metadata.entity.validation.ValidationException;
 import com.linkedin.metadata.event.EventProducer;
 import com.linkedin.metadata.key.CorpUserKey;
 import com.linkedin.metadata.models.AspectSpec;
-import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistryException;
-import com.linkedin.metadata.models.registry.MergedEntityRegistry;
+import com.linkedin.metadata.query.ListUrnsResult;
 import com.linkedin.metadata.run.AspectRowSummary;
 import com.linkedin.metadata.service.UpdateIndicesService;
 import com.linkedin.metadata.snapshot.CorpUserSnapshot;
 import com.linkedin.metadata.snapshot.Snapshot;
+import com.linkedin.metadata.utils.AuditStampUtils;
 import com.linkedin.metadata.utils.EntityKeyUtils;
 import com.linkedin.metadata.utils.GenericRecordUtils;
 import com.linkedin.mxe.GenericAspect;
@@ -75,6 +82,7 @@
 import com.linkedin.structured.StructuredPropertyValueAssignmentArray;
 import com.linkedin.util.Pair;
 import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
 import jakarta.annotation.Nonnull;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -113,18 +121,15 @@ public abstract class EntityServiceTest<T_AD extends AspectDao, T_RS extends Ret
   protected T_RS _retentionService;
 
   protected static final AuditStamp TEST_AUDIT_STAMP = AspectGenerationUtils.createAuditStamp();
-  protected final EntityRegistry _snapshotEntityRegistry = new TestEntityRegistry();
-  protected final EntityRegistry _configEntityRegistry =
-      new ConfigEntityRegistry(
-          Snapshot.class.getClassLoader().getResourceAsStream("entity-registry.yml"));
-  protected final EntityRegistry _testEntityRegistry =
-      new MergedEntityRegistry(_snapshotEntityRegistry).apply(_configEntityRegistry);
+  protected OperationContext opContext = TestOperationContexts.systemContextNoSearchAuthorization();
+  protected final EntityRegistry _testEntityRegistry = opContext.getEntityRegistry();
+  protected OperationContext userContext =
+      TestOperationContexts.userContextNoSearchAuthorization(_testEntityRegistry);
   protected EventProducer _mockProducer;
   protected UpdateIndicesService _mockUpdateIndicesService;
 
-  protected OperationContext opContext;
   protected final AspectSpec structuredPropertiesDefinitionAspect =
-      _configEntityRegistry.getAspectSpecs().get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME);
+      _testEntityRegistry.getAspectSpecs().get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME);
 
   protected EntityServiceTest() throws EntityRegistryException {}
 
@@ -498,7 +503,7 @@ public void testIngestAspectsGetLatestAspects() throws Exception {
 
     Map<String, RecordTemplate> latestAspects =
         _entityServiceImpl.getLatestAspectsForUrn(
-            opContext, entityUrn, new HashSet<>(Arrays.asList(aspectName1, aspectName2)));
+            opContext, entityUrn, new HashSet<>(Arrays.asList(aspectName1, aspectName2)), false);
     assertTrue(DataTemplateUtil.areEqual(writeAspect1, latestAspects.get(aspectName1)));
     assertTrue(DataTemplateUtil.areEqual(writeAspect2, latestAspects.get(aspectName2)));
 
@@ -554,7 +559,7 @@ public void testReingestAspectsGetLatestAspects() throws Exception {
 
     Map<String, RecordTemplate> latestAspects =
         _entityServiceImpl.getLatestAspectsForUrn(
-            opContext, entityUrn, new HashSet<>(List.of(aspectName1)));
+            opContext, entityUrn, new HashSet<>(List.of(aspectName1)), false);
     assertTrue(DataTemplateUtil.areEqual(writeAspect1, latestAspects.get(aspectName1)));
 
     verify(_mockProducer, times(1))
@@ -611,6 +616,9 @@ public void testReingestLineageAspect() throws Exception {
                 entityUrn,
                 _testEntityRegistry.getEntitySpec(entityUrn.getEntityType()).getKeyAspectSpec())));
 
+    SystemMetadata futureSystemMetadata = AspectGenerationUtils.createSystemMetadata(1);
+    futureSystemMetadata.setLastObserved(futureSystemMetadata.getLastObserved() + 1);
+
     final MetadataChangeLog restateChangeLog = new MetadataChangeLog();
     restateChangeLog.setEntityType(entityUrn.getEntityType());
     restateChangeLog.setEntityUrn(entityUrn);
@@ -618,7 +626,7 @@ public void testReingestLineageAspect() throws Exception {
     restateChangeLog.setAspectName(aspectName1);
     restateChangeLog.setCreated(TEST_AUDIT_STAMP);
     restateChangeLog.setAspect(aspect);
-    restateChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1));
+    restateChangeLog.setSystemMetadata(futureSystemMetadata);
     restateChangeLog.setPreviousAspectValue(aspect);
     restateChangeLog.setPreviousSystemMetadata(
         simulatePullFromDB(initialSystemMetadata, SystemMetadata.class));
@@ -630,7 +638,7 @@ public void testReingestLineageAspect() throws Exception {
 
     Map<String, RecordTemplate> latestAspects =
         _entityServiceImpl.getLatestAspectsForUrn(
-            opContext, entityUrn, new HashSet<>(List.of(aspectName1)));
+            opContext, entityUrn, new HashSet<>(List.of(aspectName1)), false);
     assertTrue(DataTemplateUtil.areEqual(upstreamLineage, latestAspects.get(aspectName1)));
 
     verify(_mockProducer, times(1))
@@ -642,11 +650,7 @@ public void testReingestLineageAspect() throws Exception {
     clearInvocations(_mockProducer);
 
     _entityServiceImpl.ingestAspects(
-        opContext,
-        entityUrn,
-        pairToIngest,
-        TEST_AUDIT_STAMP,
-        AspectGenerationUtils.createSystemMetadata());
+        opContext, entityUrn, pairToIngest, TEST_AUDIT_STAMP, futureSystemMetadata);
 
     verify(_mockProducer, times(1))
         .produceMetadataChangeLog(
@@ -688,6 +692,12 @@ public void testReingestLineageProposal() throws Exception {
     initialChangeLog.setAspect(genericAspect);
     initialChangeLog.setSystemMetadata(metadata1);
 
+    SystemMetadata futureSystemMetadata = AspectGenerationUtils.createSystemMetadata(1);
+    futureSystemMetadata.setLastObserved(futureSystemMetadata.getLastObserved() + 1);
+
+    MetadataChangeProposal mcp2 = new MetadataChangeProposal(mcp1.data().copy());
+    mcp2.getSystemMetadata().setLastObserved(futureSystemMetadata.getLastObserved());
+
     final MetadataChangeLog restateChangeLog = new MetadataChangeLog();
     restateChangeLog.setEntityType(entityUrn.getEntityType());
     restateChangeLog.setEntityUrn(entityUrn);
@@ -695,13 +705,13 @@ public void testReingestLineageProposal() throws Exception {
     restateChangeLog.setAspectName(aspectName1);
     restateChangeLog.setCreated(TEST_AUDIT_STAMP);
     restateChangeLog.setAspect(genericAspect);
-    restateChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1));
+    restateChangeLog.setSystemMetadata(futureSystemMetadata);
     restateChangeLog.setPreviousAspectValue(genericAspect);
     restateChangeLog.setPreviousSystemMetadata(simulatePullFromDB(metadata1, SystemMetadata.class));
 
     Map<String, RecordTemplate> latestAspects =
         _entityServiceImpl.getLatestAspectsForUrn(
-            opContext, entityUrn, new HashSet<>(List.of(aspectName1)));
+            opContext, entityUrn, new HashSet<>(List.of(aspectName1)), false);
     assertTrue(DataTemplateUtil.areEqual(upstreamLineage, latestAspects.get(aspectName1)));
 
     verify(_mockProducer, times(1))
@@ -712,7 +722,7 @@ public void testReingestLineageProposal() throws Exception {
     // unless invocations are cleared
     clearInvocations(_mockProducer);
 
-    _entityServiceImpl.ingestProposal(opContext, mcp1, TEST_AUDIT_STAMP, false);
+    _entityServiceImpl.ingestProposal(opContext, mcp2, TEST_AUDIT_STAMP, false);
 
     verify(_mockProducer, times(1))
         .produceMetadataChangeLog(
@@ -935,32 +945,32 @@ public void testRollbackAspect() throws AssertionError {
                 .recordTemplate(writeAspect1)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn2)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect2)
                 .auditStamp(TEST_AUDIT_STAMP)
                 .systemMetadata(metadata1)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn3)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect3)
                 .auditStamp(TEST_AUDIT_STAMP)
                 .systemMetadata(metadata1)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn1)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect1Overwrite)
                 .systemMetadata(metadata2)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1027,25 +1037,25 @@ public void testRollbackKey() throws AssertionError {
                 .recordTemplate(writeAspect1)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn1)
                 .aspectName(keyAspectName)
                 .recordTemplate(writeKey1)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn1)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect1Overwrite)
                 .systemMetadata(metadata2)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1120,39 +1130,39 @@ public void testRollbackUrn() throws AssertionError {
                 .recordTemplate(writeAspect1)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn1)
                 .aspectName(keyAspectName)
                 .recordTemplate(writeKey1)
                 .auditStamp(TEST_AUDIT_STAMP)
                 .systemMetadata(metadata1)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn2)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect2)
                 .auditStamp(TEST_AUDIT_STAMP)
                 .systemMetadata(metadata1)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn3)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect3)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn1)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect1Overwrite)
                 .systemMetadata(metadata2)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1198,11 +1208,11 @@ public void testIngestGetLatestAspect() throws AssertionError {
                 .recordTemplate(writeAspect1)
                 .auditStamp(TEST_AUDIT_STAMP)
                 .systemMetadata(metadata1)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1254,11 +1264,11 @@ public void testIngestGetLatestAspect() throws AssertionError {
                 .recordTemplate(writeAspect2)
                 .auditStamp(TEST_AUDIT_STAMP)
                 .systemMetadata(metadata2)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1310,11 +1320,11 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception {
                 .recordTemplate(writeAspect1)
                 .auditStamp(TEST_AUDIT_STAMP)
                 .systemMetadata(metadata1)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1337,11 +1347,11 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception {
                 .recordTemplate(writeAspect2)
                 .systemMetadata(metadata2)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1396,7 +1406,7 @@ public void testIngestSameAspect() throws AssertionError {
     SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123");
     SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456");
     SystemMetadata metadata3 =
-        AspectGenerationUtils.createSystemMetadata(1635792689, "run-123", "run-456", "1");
+        AspectGenerationUtils.createSystemMetadata(1635792689, "run-456", "run-123", "1");
 
     List<ChangeItemImpl> items =
         List.of(
@@ -1406,11 +1416,11 @@ public void testIngestSameAspect() throws AssertionError {
                 .recordTemplate(writeAspect1)
                 .systemMetadata(metadata1)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1462,11 +1472,11 @@ public void testIngestSameAspect() throws AssertionError {
                 .recordTemplate(writeAspect2)
                 .systemMetadata(metadata2)
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1488,6 +1498,9 @@ public void testIngestSameAspect() throws AssertionError {
 
     assertTrue(
         DataTemplateUtil.areEqual(
+            EntityApiUtils.parseSystemMetadata(readAspectDao2.getSystemMetadata()), metadata3),
+        String.format(
+            "Expected %s == %s",
             EntityApiUtils.parseSystemMetadata(readAspectDao2.getSystemMetadata()), metadata3));
 
     verify(_mockProducer, times(0))
@@ -1521,46 +1534,46 @@ public void testRetention() throws AssertionError {
                 .recordTemplate(writeAspect1)
                 .systemMetadata(AspectGenerationUtils.createSystemMetadata())
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect1a)
                 .systemMetadata(AspectGenerationUtils.createSystemMetadata())
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn)
                 .aspectName(aspectName)
                 .recordTemplate(writeAspect1b)
                 .systemMetadata(AspectGenerationUtils.createSystemMetadata())
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn)
                 .aspectName(aspectName2)
                 .recordTemplate(writeAspect2)
                 .systemMetadata(AspectGenerationUtils.createSystemMetadata())
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn)
                 .aspectName(aspectName2)
                 .recordTemplate(writeAspect2a)
                 .systemMetadata(AspectGenerationUtils.createSystemMetadata())
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn)
                 .aspectName(aspectName2)
                 .recordTemplate(writeAspect2b)
                 .systemMetadata(AspectGenerationUtils.createSystemMetadata())
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1597,18 +1610,18 @@ public void testRetention() throws AssertionError {
                 .recordTemplate(writeAspect1c)
                 .systemMetadata(AspectGenerationUtils.createSystemMetadata())
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()),
+                .build(opContext.getAspectRetriever()),
             ChangeItemImpl.builder()
                 .urn(entityUrn)
                 .aspectName(aspectName2)
                 .recordTemplate(writeAspect2c)
                 .systemMetadata(AspectGenerationUtils.createSystemMetadata())
                 .auditStamp(TEST_AUDIT_STAMP)
-                .build(opContext.getAspectRetrieverOpt().get()));
+                .build(opContext.getAspectRetriever()));
     _entityServiceImpl.ingestAspects(
         opContext,
         AspectsBatchImpl.builder()
-            .retrieverContext(opContext.getRetrieverContext().get())
+            .retrieverContext(opContext.getRetrieverContext())
             .items(items)
             .build(),
         true,
@@ -1842,7 +1855,7 @@ public void testValidateUrn() throws Exception {
       ValidationApiUtils.validateUrn(opContext.getEntityRegistry(), urnWithMismatchedParens);
       Assert.fail("Should have raised IllegalArgumentException for URN with mismatched parens");
     } catch (IllegalArgumentException e) {
-      assertTrue(e.getMessage().contains("mismatched paren nesting"));
+      assertTrue(e.getMessage().contains("[test(Key]"));
     }
 
     Urn invalidType = new Urn("li", "fakeMadeUpType", new TupleKey("testKey"));
@@ -1969,8 +1982,7 @@ public void testStructuredPropertyIngestProposal() throws Exception {
           stream
               .map(
                   entityAspect ->
-                      EntityUtils.toSystemAspect(
-                              opContext.getRetrieverContext().get(), entityAspect)
+                      EntityUtils.toSystemAspect(opContext.getRetrieverContext(), entityAspect)
                           .get()
                           .getAspect(StructuredPropertyDefinition.class))
               .collect(Collectors.toSet());
@@ -1982,7 +1994,10 @@ public void testStructuredPropertyIngestProposal() throws Exception {
     SystemEntityClient mockSystemEntityClient = Mockito.mock(SystemEntityClient.class);
     Mockito.when(
             mockSystemEntityClient.getLatestAspectObject(
-                any(OperationContext.class), eq(firstPropertyUrn), eq("propertyDefinition")))
+                any(OperationContext.class),
+                eq(firstPropertyUrn),
+                eq("propertyDefinition"),
+                anyBoolean()))
         .thenReturn(new com.linkedin.entity.Aspect(structuredPropertyDefinition.data()));
 
     // Add a value for that property
@@ -2049,8 +2064,7 @@ public void testStructuredPropertyIngestProposal() throws Exception {
           stream
               .map(
                   entityAspect ->
-                      EntityUtils.toSystemAspect(
-                              opContext.getRetrieverContext().get(), entityAspect)
+                      EntityUtils.toSystemAspect(opContext.getRetrieverContext(), entityAspect)
                           .get()
                           .getAspect(StructuredPropertyDefinition.class))
               .collect(Collectors.toSet());
@@ -2061,7 +2075,10 @@ public void testStructuredPropertyIngestProposal() throws Exception {
 
     Mockito.when(
             mockSystemEntityClient.getLatestAspectObject(
-                any(OperationContext.class), eq(secondPropertyUrn), eq("propertyDefinition")))
+                any(OperationContext.class),
+                eq(secondPropertyUrn),
+                eq("propertyDefinition"),
+                anyBoolean()))
         .thenReturn(new com.linkedin.entity.Aspect(secondDefinition.data()));
 
     // Get existing value for first structured property
@@ -2145,7 +2162,7 @@ public void testCreateChangeTypeProposal() {
         ValidationException.class,
         () ->
             _entityServiceImpl.ingestProposal(
-                opContext, secondCreateProposal, TEST_AUDIT_STAMP, false));
+                userContext, secondCreateProposal, TEST_AUDIT_STAMP, false));
   }
 
   @Test
@@ -2185,6 +2202,474 @@ public void testExists() throws Exception {
         Set.of(existentUrn, noStatusUrn, softDeletedUrn));
   }
 
+  @Test
+  public void testBatchDuplicate() throws Exception {
+    Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:batchDuplicateTest");
+    SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata();
+    ChangeItemImpl item1 =
+        ChangeItemImpl.builder()
+            .urn(entityUrn)
+            .aspectName(STATUS_ASPECT_NAME)
+            .recordTemplate(new Status().setRemoved(true))
+            .systemMetadata(systemMetadata.copy())
+            .auditStamp(TEST_AUDIT_STAMP)
+            .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
+    ChangeItemImpl item2 =
+        ChangeItemImpl.builder()
+            .urn(entityUrn)
+            .aspectName(STATUS_ASPECT_NAME)
+            .recordTemplate(new Status().setRemoved(false))
+            .systemMetadata(systemMetadata.copy())
+            .auditStamp(TEST_AUDIT_STAMP)
+            .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
+    _entityServiceImpl.ingestAspects(
+        opContext,
+        AspectsBatchImpl.builder()
+            .retrieverContext(opContext.getRetrieverContext())
+            .items(List.of(item1, item2))
+            .build(),
+        false,
+        true);
+
+    // List aspects urns
+    ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 2);
+
+    assertEquals(batch.getStart().intValue(), 0);
+    assertEquals(batch.getCount().intValue(), 1);
+    assertEquals(batch.getTotal().intValue(), 1);
+    assertEquals(batch.getEntities().size(), 1);
+    assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString());
+
+    EnvelopedAspect envelopedAspect =
+        _entityServiceImpl.getLatestEnvelopedAspect(
+            opContext, CORP_USER_ENTITY_NAME, entityUrn, STATUS_ASPECT_NAME);
+    assertEquals(
+        envelopedAspect.getSystemMetadata().getVersion(),
+        "2",
+        "Expected version 2 after accounting for sequential duplicates");
+    assertEquals(
+        envelopedAspect.getValue().toString(),
+        "{removed=false}",
+        "Expected 2nd item to be the latest");
+  }
+
+  @Test
+  public void testBatchPatchWithTrailingNoOp() throws Exception {
+    Urn entityUrn =
+        UrnUtils.getUrn(
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchWithTrailingNoOp,PROD)");
+    TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1");
+    Urn tag2 = UrnUtils.getUrn("urn:li:tag:tag2");
+    Urn tagOther = UrnUtils.getUrn("urn:li:tag:other");
+
+    SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata();
+
+    ChangeItemImpl initialAspectTag1 =
+        ChangeItemImpl.builder()
+            .urn(entityUrn)
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .recordTemplate(
+                new GlobalTags()
+                    .setTags(new TagAssociationArray(new TagAssociation().setTag(tag1))))
+            .systemMetadata(systemMetadata.copy())
+            .auditStamp(TEST_AUDIT_STAMP)
+            .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
+
+    PatchItemImpl patchAdd2 =
+        PatchItemImpl.builder()
+            .urn(entityUrn)
+            .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME))
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .aspectSpec(
+                _testEntityRegistry
+                    .getEntitySpec(DATASET_ENTITY_NAME)
+                    .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME))
+            .patch(
+                GenericJsonPatch.builder()
+                    .arrayPrimaryKeys(Map.of("properties", List.of("tag")))
+                    .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2)))
+                    .build()
+                    .getJsonPatch())
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(_testEntityRegistry);
+
+    PatchItemImpl patchRemoveNonExistent =
+        PatchItemImpl.builder()
+            .urn(entityUrn)
+            .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME))
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .aspectSpec(
+                _testEntityRegistry
+                    .getEntitySpec(DATASET_ENTITY_NAME)
+                    .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME))
+            .patch(
+                GenericJsonPatch.builder()
+                    .arrayPrimaryKeys(Map.of("properties", List.of("tag")))
+                    .patch(List.of(tagPatchOp(PatchOperationType.REMOVE, tagOther)))
+                    .build()
+                    .getJsonPatch())
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(_testEntityRegistry);
+
+    // establish base entity
+    _entityServiceImpl.ingestAspects(
+        opContext,
+        AspectsBatchImpl.builder()
+            .retrieverContext(opContext.getRetrieverContext())
+            .items(List.of(initialAspectTag1))
+            .build(),
+        false,
+        true);
+
+    _entityServiceImpl.ingestAspects(
+        opContext,
+        AspectsBatchImpl.builder()
+            .retrieverContext(opContext.getRetrieverContext())
+            .items(List.of(patchAdd2, patchRemoveNonExistent))
+            .build(),
+        false,
+        true);
+
+    // List aspects urns
+    ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1);
+
+    assertEquals(batch.getStart().intValue(), 0);
+    assertEquals(batch.getCount().intValue(), 1);
+    assertEquals(batch.getTotal().intValue(), 1);
+    assertEquals(batch.getEntities().size(), 1);
+    assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString());
+
+    EnvelopedAspect envelopedAspect =
+        _entityServiceImpl.getLatestEnvelopedAspect(
+            opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME);
+    assertEquals(
+        envelopedAspect.getSystemMetadata().getVersion(),
+        "2",
+        "Expected version 3. 1 - Initial, + 1 add, 1 remove");
+    assertEquals(
+        new GlobalTags(envelopedAspect.getValue().data())
+            .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()),
+        Set.of(tag1, tag2),
+        "Expected both tags");
+  }
+
+  @Test
+  public void testBatchPatchAdd() throws Exception {
+    Urn entityUrn =
+        UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchAdd,PROD)");
+    TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1");
+    TagUrn tag2 = TagUrn.createFromString("urn:li:tag:tag2");
+    TagUrn tag3 = TagUrn.createFromString("urn:li:tag:tag3");
+
+    SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata();
+
+    ChangeItemImpl initialAspectTag1 =
+        ChangeItemImpl.builder()
+            .urn(entityUrn)
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .recordTemplate(
+                new GlobalTags()
+                    .setTags(new TagAssociationArray(new TagAssociation().setTag(tag1))))
+            .systemMetadata(systemMetadata.copy())
+            .auditStamp(TEST_AUDIT_STAMP)
+            .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
+
+    PatchItemImpl patchAdd3 =
+        PatchItemImpl.builder()
+            .urn(entityUrn)
+            .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME))
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .aspectSpec(
+                _testEntityRegistry
+                    .getEntitySpec(DATASET_ENTITY_NAME)
+                    .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME))
+            .patch(
+                GenericJsonPatch.builder()
+                    .arrayPrimaryKeys(Map.of("properties", List.of("tag")))
+                    .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag3)))
+                    .build()
+                    .getJsonPatch())
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(_testEntityRegistry);
+
+    PatchItemImpl patchAdd2 =
+        PatchItemImpl.builder()
+            .urn(entityUrn)
+            .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME))
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .aspectSpec(
+                _testEntityRegistry
+                    .getEntitySpec(DATASET_ENTITY_NAME)
+                    .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME))
+            .patch(
+                GenericJsonPatch.builder()
+                    .arrayPrimaryKeys(Map.of("properties", List.of("tag")))
+                    .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2)))
+                    .build()
+                    .getJsonPatch())
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(_testEntityRegistry);
+
+    PatchItemImpl patchAdd1 =
+        PatchItemImpl.builder()
+            .urn(entityUrn)
+            .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME))
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .aspectSpec(
+                _testEntityRegistry
+                    .getEntitySpec(DATASET_ENTITY_NAME)
+                    .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME))
+            .patch(
+                GenericJsonPatch.builder()
+                    .arrayPrimaryKeys(Map.of("properties", List.of("tag")))
+                    .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag1)))
+                    .build()
+                    .getJsonPatch())
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(_testEntityRegistry);
+
+    // establish base entity
+    _entityServiceImpl.ingestAspects(
+        opContext,
+        AspectsBatchImpl.builder()
+            .retrieverContext(opContext.getRetrieverContext())
+            .items(List.of(initialAspectTag1))
+            .build(),
+        false,
+        true);
+
+    _entityServiceImpl.ingestAspects(
+        opContext,
+        AspectsBatchImpl.builder()
+            .retrieverContext(opContext.getRetrieverContext())
+            .items(List.of(patchAdd3, patchAdd2, patchAdd1))
+            .build(),
+        false,
+        true);
+
+    // List aspects urns
+    ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1);
+
+    assertEquals(batch.getStart().intValue(), 0);
+    assertEquals(batch.getCount().intValue(), 1);
+    assertEquals(batch.getTotal().intValue(), 1);
+    assertEquals(batch.getEntities().size(), 1);
+    assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString());
+
+    EnvelopedAspect envelopedAspect =
+        _entityServiceImpl.getLatestEnvelopedAspect(
+            opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME);
+    assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "3", "Expected version 4");
+    assertEquals(
+        new GlobalTags(envelopedAspect.getValue().data())
+            .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()),
+        Set.of(tag1, tag2, tag3),
+        "Expected all tags");
+  }
+
+  @Test
+  public void testBatchPatchAddDuplicate() throws Exception {
+    Urn entityUrn =
+        UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchAdd,PROD)");
+    List<TagAssociation> initialTags =
+        List.of(
+                TagUrn.createFromString("urn:li:tag:__default_large_table"),
+                TagUrn.createFromString("urn:li:tag:__default_low_queries"),
+                TagUrn.createFromString("urn:li:tag:__default_low_changes"),
+                TagUrn.createFromString("urn:li:tag:!10TB+ tables"))
+            .stream()
+            .map(tag -> new TagAssociation().setTag(tag))
+            .collect(Collectors.toList());
+    TagUrn tag2 = TagUrn.createFromString("urn:li:tag:$ 1TB+");
+
+    SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata();
+
+    SystemMetadata patchSystemMetadata = new SystemMetadata();
+    patchSystemMetadata.setLastObserved(systemMetadata.getLastObserved() + 1);
+    patchSystemMetadata.setProperties(new StringMap(Map.of(APP_SOURCE, METADATA_TESTS_SOURCE)));
+
+    ChangeItemImpl initialAspectTag1 =
+        ChangeItemImpl.builder()
+            .urn(entityUrn)
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .recordTemplate(new GlobalTags().setTags(new TagAssociationArray(initialTags)))
+            .systemMetadata(systemMetadata.copy())
+            .auditStamp(TEST_AUDIT_STAMP)
+            .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
+
+    PatchItemImpl patchAdd2 =
+        PatchItemImpl.builder()
+            .urn(entityUrn)
+            .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME))
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .aspectSpec(
+                _testEntityRegistry
+                    .getEntitySpec(DATASET_ENTITY_NAME)
+                    .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME))
+            .patch(
+                GenericJsonPatch.builder()
+                    .arrayPrimaryKeys(Map.of("properties", List.of("tag")))
+                    .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2)))
+                    .build()
+                    .getJsonPatch())
+            .systemMetadata(patchSystemMetadata)
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(_testEntityRegistry);
+
+    // establish base entity
+    _entityServiceImpl.ingestAspects(
+        opContext,
+        AspectsBatchImpl.builder()
+            .retrieverContext(opContext.getRetrieverContext())
+            .items(List.of(initialAspectTag1))
+            .build(),
+        false,
+        true);
+
+    _entityServiceImpl.ingestAspects(
+        opContext,
+        AspectsBatchImpl.builder()
+            .retrieverContext(opContext.getRetrieverContext())
+            .items(List.of(patchAdd2, patchAdd2)) // duplicate
+            .build(),
+        false,
+        true);
+
+    // List aspects urns
+    ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1);
+
+    assertEquals(batch.getStart().intValue(), 0);
+    assertEquals(batch.getCount().intValue(), 1);
+    assertEquals(batch.getTotal().intValue(), 1);
+    assertEquals(batch.getEntities().size(), 1);
+    assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString());
+
+    EnvelopedAspect envelopedAspect =
+        _entityServiceImpl.getLatestEnvelopedAspect(
+            opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME);
+    assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "2", "Expected version 2");
+    assertEquals(
+        new GlobalTags(envelopedAspect.getValue().data())
+            .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()),
+        Stream.concat(initialTags.stream().map(TagAssociation::getTag), Stream.of(tag2))
+            .collect(Collectors.toSet()),
+        "Expected all tags");
+  }
+
+  @Test
+  public void testPatchRemoveNonExistent() throws Exception {
+    Urn entityUrn =
+        UrnUtils.getUrn(
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,testPatchRemoveNonExistent,PROD)");
+    TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1");
+
+    PatchItemImpl patchRemove =
+        PatchItemImpl.builder()
+            .urn(entityUrn)
+            .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME))
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .aspectSpec(
+                _testEntityRegistry
+                    .getEntitySpec(DATASET_ENTITY_NAME)
+                    .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME))
+            .patch(
+                GenericJsonPatch.builder()
+                    .arrayPrimaryKeys(Map.of("properties", List.of("tag")))
+                    .patch(List.of(tagPatchOp(PatchOperationType.REMOVE, tag1)))
+                    .build()
+                    .getJsonPatch())
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(_testEntityRegistry);
+
+    List<UpdateAspectResult> results =
+        _entityServiceImpl.ingestAspects(
+            opContext,
+            AspectsBatchImpl.builder()
+                .retrieverContext(opContext.getRetrieverContext())
+                .items(List.of(patchRemove))
+                .build(),
+            false,
+            true);
+
+    assertEquals(results.size(), 4, "Expected default aspects + empty globalTags");
+
+    // List aspects urns
+    ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1);
+
+    assertEquals(batch.getStart().intValue(), 0);
+    assertEquals(batch.getCount().intValue(), 1);
+    assertEquals(batch.getTotal().intValue(), 1);
+    assertEquals(batch.getEntities().size(), 1);
+    assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString());
+
+    EnvelopedAspect envelopedAspect =
+        _entityServiceImpl.getLatestEnvelopedAspect(
+            opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME);
+    assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "1", "Expected version 4");
+    assertEquals(
+        new GlobalTags(envelopedAspect.getValue().data())
+            .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()),
+        Set.of(),
+        "Expected empty tags");
+  }
+
+  @Test
+  public void testPatchAddNonExistent() throws Exception {
+    Urn entityUrn =
+        UrnUtils.getUrn(
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,testPatchAddNonExistent,PROD)");
+    TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1");
+
+    PatchItemImpl patchAdd =
+        PatchItemImpl.builder()
+            .urn(entityUrn)
+            .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME))
+            .aspectName(GLOBAL_TAGS_ASPECT_NAME)
+            .aspectSpec(
+                _testEntityRegistry
+                    .getEntitySpec(DATASET_ENTITY_NAME)
+                    .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME))
+            .patch(
+                GenericJsonPatch.builder()
+                    .arrayPrimaryKeys(Map.of("properties", List.of("tag")))
+                    .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag1)))
+                    .build()
+                    .getJsonPatch())
+            .auditStamp(AuditStampUtils.createDefaultAuditStamp())
+            .build(_testEntityRegistry);
+
+    List<UpdateAspectResult> results =
+        _entityServiceImpl.ingestAspects(
+            opContext,
+            AspectsBatchImpl.builder()
+                .retrieverContext(opContext.getRetrieverContext())
+                .items(List.of(patchAdd))
+                .build(),
+            false,
+            true);
+
+    assertEquals(results.size(), 4, "Expected default aspects + globalTags");
+
+    // List aspects urns
+    ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1);
+
+    assertEquals(batch.getStart().intValue(), 0);
+    assertEquals(batch.getCount().intValue(), 1);
+    assertEquals(batch.getTotal().intValue(), 1);
+    assertEquals(batch.getEntities().size(), 1);
+    assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString());
+
+    EnvelopedAspect envelopedAspect =
+        _entityServiceImpl.getLatestEnvelopedAspect(
+            opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME);
+    assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "1", "Expected version 4");
+    assertEquals(
+        new GlobalTags(envelopedAspect.getValue().data())
+            .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()),
+        Set.of(tag1),
+        "Expected all tags");
+  }
+
   @Nonnull
   protected com.linkedin.entity.Entity createCorpUserEntity(Urn entityUrn, String email)
       throws Exception {
@@ -2216,4 +2701,14 @@ protected <T extends RecordTemplate> Pair<String, RecordTemplate> getAspectRecor
         RecordUtils.toRecordTemplate(clazz, objectMapper.writeValueAsString(aspect));
     return new Pair<>(AspectGenerationUtils.getAspectName(aspect), recordTemplate);
   }
+
+  private static GenericJsonPatch.PatchOp tagPatchOp(PatchOperationType op, Urn tagUrn) {
+    GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp();
+    patchOp.setOp(op.getValue());
+    patchOp.setPath(String.format("/tags/%s", tagUrn));
+    if (PatchOperationType.ADD.equals(op)) {
+      patchOp.setValue(Map.of("tag", tagUrn.toString()));
+    }
+    return patchOp;
+  }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/cassandra/CassandraEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/cassandra/CassandraEntityServiceTest.java
index 5535866f87c99..b4fbfecc9d60d 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/entity/cassandra/CassandraEntityServiceTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/cassandra/CassandraEntityServiceTest.java
@@ -10,11 +10,13 @@
 import com.linkedin.metadata.AspectGenerationUtils;
 import com.linkedin.metadata.AspectIngestionUtils;
 import com.linkedin.metadata.CassandraTestUtils;
+import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.config.PreProcessHooks;
 import com.linkedin.metadata.entity.EntityServiceAspectRetriever;
 import com.linkedin.metadata.entity.EntityServiceImpl;
 import com.linkedin.metadata.entity.EntityServiceTest;
 import com.linkedin.metadata.entity.ListResult;
+import com.linkedin.metadata.entity.SearchRetriever;
 import com.linkedin.metadata.event.EventProducer;
 import com.linkedin.metadata.key.CorpUserKey;
 import com.linkedin.metadata.models.registry.EntityRegistryException;
@@ -93,13 +95,17 @@ private void configureComponents() {
                             .entityService(_entityServiceImpl)
                             .entityRegistry(_testEntityRegistry)
                             .build())
-                    .graphRetriever(TestOperationContexts.emptyGraphRetriever)
-                    .searchRetriever(TestOperationContexts.emptySearchRetriever)
+                    .cachingAspectRetriever(
+                        TestOperationContexts.emptyActiveUsersAspectRetriever(
+                            () -> _testEntityRegistry))
+                    .graphRetriever(GraphRetriever.EMPTY)
+                    .searchRetriever(SearchRetriever.EMPTY)
                     .build(),
             null,
             opContext ->
-                ((EntityServiceAspectRetriever) opContext.getAspectRetrieverOpt().get())
-                    .setSystemOperationContext(opContext));
+                ((EntityServiceAspectRetriever) opContext.getAspectRetriever())
+                    .setSystemOperationContext(opContext),
+            null);
   }
 
   /**
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java
index 775770d28b4a2..4915f89783596 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java
@@ -1,5 +1,7 @@
 package com.linkedin.metadata.entity.ebean;
 
+import static com.linkedin.metadata.Constants.ASPECT_LATEST_VERSION;
+import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME;
 import static org.mockito.Mockito.mock;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
@@ -9,6 +11,7 @@
 import com.linkedin.metadata.EbeanTestUtils;
 import com.linkedin.metadata.aspect.batch.AspectsBatch;
 import com.linkedin.metadata.config.EbeanConfiguration;
+import com.linkedin.metadata.entity.EntityAspectIdentifier;
 import io.ebean.Database;
 import io.ebean.test.LoggedSql;
 import java.util.List;
@@ -73,4 +76,40 @@ public void testGetLatestAspectsForUpdate() throws JsonProcessingException {
     assertTrue(
         sql.get(0).contains("for update;"), String.format("Did not find `for update` in %s ", sql));
   }
+
+  @Test
+  public void testbatchGetForUpdate() throws JsonProcessingException {
+    LoggedSql.start();
+
+    testDao.runInTransactionWithRetryUnlocked(
+        (txContext) -> {
+          testDao.batchGet(
+              Set.of(
+                  new EntityAspectIdentifier(
+                      "urn:li:corpuser:testbatchGetForUpdate1",
+                      DATA_PLATFORM_INSTANCE_ASPECT_NAME,
+                      ASPECT_LATEST_VERSION),
+                  new EntityAspectIdentifier(
+                      "urn:li:corpuser:testbatchGetForUpdate2",
+                      DATA_PLATFORM_INSTANCE_ASPECT_NAME,
+                      ASPECT_LATEST_VERSION)),
+              true);
+          return "";
+        },
+        mock(AspectsBatch.class),
+        0);
+
+    // Get the captured SQL statements
+    List<String> sql =
+        LoggedSql.stop().stream()
+            .filter(
+                str ->
+                    str.contains("testbatchGetForUpdate1")
+                        && str.contains("testbatchGetForUpdate2"))
+            .toList();
+    assertEquals(
+        sql.size(), 1, String.format("Found: %s", new ObjectMapper().writeValueAsString(sql)));
+    assertTrue(
+        sql.get(0).contains("FOR UPDATE;"), String.format("Did not find `for update` in %s ", sql));
+  }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImplTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImplTest.java
new file mode 100644
index 0000000000000..0a867ae3c8f2e
--- /dev/null
+++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImplTest.java
@@ -0,0 +1,41 @@
+package com.linkedin.metadata.entity.ebean.batch;
+
+import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME;
+import static org.testng.Assert.assertFalse;
+
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.Status;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.metadata.AspectGenerationUtils;
+import com.linkedin.mxe.SystemMetadata;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
+import org.testng.annotations.Test;
+
+public class ChangeItemImplTest {
+  private static final AuditStamp TEST_AUDIT_STAMP = AspectGenerationUtils.createAuditStamp();
+
+  @Test
+  public void testBatchDuplicate() throws Exception {
+    Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:batchDuplicateTest");
+    SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata();
+    ChangeItemImpl item1 =
+        ChangeItemImpl.builder()
+            .urn(entityUrn)
+            .aspectName(STATUS_ASPECT_NAME)
+            .recordTemplate(new Status().setRemoved(true))
+            .systemMetadata(systemMetadata.copy())
+            .auditStamp(TEST_AUDIT_STAMP)
+            .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
+    ChangeItemImpl item2 =
+        ChangeItemImpl.builder()
+            .urn(entityUrn)
+            .aspectName(STATUS_ASPECT_NAME)
+            .recordTemplate(new Status().setRemoved(false))
+            .systemMetadata(systemMetadata.copy())
+            .auditStamp(TEST_AUDIT_STAMP)
+            .build(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
+
+    assertFalse(item1.isDatabaseDuplicateOf(item2));
+  }
+}
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/recommendation/RecommendationsServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/recommendation/RecommendationsServiceTest.java
index ca42f0327c86d..8f68f119cb0b7 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/recommendation/RecommendationsServiceTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/recommendation/RecommendationsServiceTest.java
@@ -11,6 +11,7 @@
 import com.linkedin.metadata.recommendation.ranker.SimpleRecommendationRanker;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.net.URISyntaxException;
+import java.nio.file.AccessDeniedException;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.testng.annotations.Test;
@@ -74,7 +75,7 @@ private List<RecommendationContent> getContentFromUrns(List<Urn> urns) {
   }
 
   @Test
-  public void testService() throws URISyntaxException {
+  public void testService() throws URISyntaxException, AccessDeniedException {
     // Test non-eligible and empty
     RecommendationsService service =
         new RecommendationsService(ImmutableList.of(nonEligibleSource, emptySource), ranker);
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/schemafields/sideeffects/SchemaFieldSideEffectTest.java b/metadata-io/src/test/java/com/linkedin/metadata/schemafields/sideeffects/SchemaFieldSideEffectTest.java
index 6139776702c71..fa895cb454011 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/schemafields/sideeffects/SchemaFieldSideEffectTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/schemafields/sideeffects/SchemaFieldSideEffectTest.java
@@ -21,7 +21,8 @@
 import com.linkedin.data.ByteString;
 import com.linkedin.entity.Aspect;
 import com.linkedin.events.metadata.ChangeType;
-import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
+import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.aspect.batch.MCLItem;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
@@ -46,7 +47,6 @@
 import com.linkedin.test.metadata.aspect.TestEntityRegistry;
 import com.linkedin.test.metadata.aspect.batch.TestMCP;
 import io.datahubproject.metadata.context.RetrieverContext;
-import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -87,18 +87,18 @@ public class SchemaFieldSideEffectTest {
                       .build()))
           .build();
 
-  private AspectRetriever mockAspectRetriever;
+  private CachingAspectRetriever mockAspectRetriever;
   private RetrieverContext retrieverContext;
 
   @BeforeMethod
   public void setup() {
-    mockAspectRetriever = mock(AspectRetriever.class);
+    mockAspectRetriever = mock(CachingAspectRetriever.class);
     when(mockAspectRetriever.getEntityRegistry()).thenReturn(TEST_REGISTRY);
     retrieverContext =
         RetrieverContext.builder()
             .searchRetriever(mock(SearchRetriever.class))
-            .aspectRetriever(mockAspectRetriever)
-            .graphRetriever(TestOperationContexts.emptyGraphRetriever)
+            .cachingAspectRetriever(mockAspectRetriever)
+            .graphRetriever(GraphRetriever.EMPTY)
             .build();
   }
 
@@ -151,7 +151,7 @@ public void schemaMetadataToSchemaFieldKeyTest() {
                           UrnUtils.getUrn(
                               "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)"))
                       .aspectName(SCHEMA_FIELD_ALIASES_ASPECT)
-                      .changeType(changeType)
+                      .changeType(ChangeType.UPSERT)
                       .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME))
                       .aspectSpec(
                           TEST_REGISTRY
@@ -172,7 +172,7 @@ public void schemaMetadataToSchemaFieldKeyTest() {
                           UrnUtils.getUrn(
                               "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)"))
                       .aspectName(SCHEMA_FIELD_ALIASES_ASPECT)
-                      .changeType(changeType)
+                      .changeType(ChangeType.UPSERT)
                       .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME))
                       .aspectSpec(
                           TEST_REGISTRY
@@ -248,7 +248,7 @@ public void statusToSchemaFieldStatusTest() {
                           UrnUtils.getUrn(
                               "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)"))
                       .aspectName(STATUS_ASPECT_NAME)
-                      .changeType(changeType)
+                      .changeType(ChangeType.UPSERT)
                       .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME))
                       .aspectSpec(
                           TEST_REGISTRY
@@ -263,7 +263,7 @@ public void statusToSchemaFieldStatusTest() {
                           UrnUtils.getUrn(
                               "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)"))
                       .aspectName(STATUS_ASPECT_NAME)
-                      .changeType(changeType)
+                      .changeType(ChangeType.UPSERT)
                       .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME))
                       .aspectSpec(
                           TEST_REGISTRY
@@ -324,7 +324,7 @@ public void statusToSchemaFieldStatusTest() {
                           UrnUtils.getUrn(
                               "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)"))
                       .aspectName(STATUS_ASPECT_NAME)
-                      .changeType(changeType)
+                      .changeType(ChangeType.UPSERT)
                       .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME))
                       .aspectSpec(
                           TEST_REGISTRY
@@ -339,7 +339,7 @@ public void statusToSchemaFieldStatusTest() {
                           UrnUtils.getUrn(
                               "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)"))
                       .aspectName(STATUS_ASPECT_NAME)
-                      .changeType(changeType)
+                      .changeType(ChangeType.UPSERT)
                       .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME))
                       .aspectSpec(
                           TEST_REGISTRY
@@ -354,7 +354,7 @@ public void statusToSchemaFieldStatusTest() {
                           UrnUtils.getUrn(
                               "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)"))
                       .aspectName(SCHEMA_FIELD_ALIASES_ASPECT)
-                      .changeType(changeType)
+                      .changeType(ChangeType.UPSERT)
                       .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME))
                       .aspectSpec(
                           TEST_REGISTRY
@@ -375,7 +375,7 @@ public void statusToSchemaFieldStatusTest() {
                           UrnUtils.getUrn(
                               "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)"))
                       .aspectName(SCHEMA_FIELD_ALIASES_ASPECT)
-                      .changeType(changeType)
+                      .changeType(ChangeType.UPSERT)
                       .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME))
                       .aspectSpec(
                           TEST_REGISTRY
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java
index bc3c892e07b1b..476f0114817be 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java
@@ -283,7 +283,7 @@ public void testFixtureInitialization() {
         Map.of(
             "dataset", 13,
             "chart", 0,
-            "container", 1,
+            "container", 2,
             "dashboard", 0,
             "tag", 0,
             "mlmodel", 0);
@@ -903,6 +903,26 @@ public void testContainerAutoComplete() {
             });
   }
 
+  @Test
+  public void testContainerAutoComplete_with_exactMatch_onTop() {
+    List.of("container")
+        .forEach(
+            query -> {
+              try {
+                AutoCompleteResults result =
+                    autocomplete(
+                        getOperationContext(), new ContainerType(getEntityClient()), query);
+                assertTrue(
+                    result.getSuggestions().get(0).equals("container"),
+                    String.format(
+                        "Expected query:`%s` on top of suggestions, found %s",
+                        query, result.getSuggestions().get(0)));
+              } catch (Exception e) {
+                throw new RuntimeException(e);
+              }
+            });
+  }
+
   @Test
   public void testGroupAutoComplete() {
     List.of("T", "Te", "Tes", "Test ", "Test G", "Test Gro", "Test Group ")
@@ -1104,9 +1124,10 @@ public void testFacets() {
   @Test
   public void testNestedAggregation() {
     Set<String> expectedFacets = Set.of("platform");
+    OperationContext context =
+        getOperationContext().withSearchFlags(flags -> flags.setIncludeDefaultFacets(false));
     SearchResult testResult =
-        facetAcrossEntities(
-            getOperationContext(), getSearchService(), "cypress", List.copyOf(expectedFacets));
+        facetAcrossEntities(context, getSearchService(), "cypress", List.copyOf(expectedFacets));
     assertEquals(testResult.getMetadata().getAggregations().size(), 1);
     expectedFacets.forEach(
         facet -> {
@@ -1123,8 +1144,7 @@ public void testNestedAggregation() {
 
     expectedFacets = Set.of("platform", "typeNames", "_entityType", "entity");
     SearchResult testResult2 =
-        facetAcrossEntities(
-            getOperationContext(), getSearchService(), "cypress", List.copyOf(expectedFacets));
+        facetAcrossEntities(context, getSearchService(), "cypress", List.copyOf(expectedFacets));
     assertEquals(testResult2.getMetadata().getAggregations().size(), 4);
     expectedFacets.forEach(
         facet -> {
@@ -1171,8 +1191,7 @@ public void testNestedAggregation() {
 
     expectedFacets = Set.of("platform", "typeNames", "entity");
     SearchResult testResult3 =
-        facetAcrossEntities(
-            getOperationContext(), getSearchService(), "cypress", List.copyOf(expectedFacets));
+        facetAcrossEntities(context, getSearchService(), "cypress", List.copyOf(expectedFacets));
     assertEquals(testResult3.getMetadata().getAggregations().size(), 4);
     expectedFacets.forEach(
         facet -> {
@@ -1202,8 +1221,7 @@ public void testNestedAggregation() {
     String singleNestedFacet = String.format("_entityType%sowners", AGGREGATION_SEPARATOR_CHAR);
     expectedFacets = Set.of(singleNestedFacet);
     SearchResult testResultSingleNested =
-        facetAcrossEntities(
-            getOperationContext(), getSearchService(), "cypress", List.copyOf(expectedFacets));
+        facetAcrossEntities(context, getSearchService(), "cypress", List.copyOf(expectedFacets));
     assertEquals(testResultSingleNested.getMetadata().getAggregations().size(), 1);
     Map<String, Long> expectedNestedFacetCounts = new HashMap<>();
     expectedNestedFacetCounts.put("datajob␞urn:li:corpuser:datahub", 2L);
@@ -1225,8 +1243,7 @@ public void testNestedAggregation() {
 
     expectedFacets = Set.of("platform", singleNestedFacet, "typeNames", "origin");
     SearchResult testResultNested =
-        facetAcrossEntities(
-            getOperationContext(), getSearchService(), "cypress", List.copyOf(expectedFacets));
+        facetAcrossEntities(context, getSearchService(), "cypress", List.copyOf(expectedFacets));
     assertEquals(testResultNested.getMetadata().getAggregations().size(), 4);
     expectedFacets.forEach(
         facet -> {
@@ -1347,8 +1364,8 @@ public void testScrollAcrossEntities() throws IOException {
       resultUrns.addAll(result.getEntities().stream().map(SearchEntity::getEntity).toList());
       scrollId = result.getScrollId();
     } while (scrollId != null);
-    // expect 2 total matching results
-    assertEquals(totalResults, 2, String.format("query `%s` Results: %s", query, resultUrns));
+    // expect 8 total matching results
+    assertEquals(totalResults, 8, String.format("query `%s` Results: %s", query, resultUrns));
   }
 
   @Test
@@ -1725,7 +1742,7 @@ public void testOr() {
         String.format("%s - Expected search results to include matched fields", query));
     assertEquals(
         result.getEntities().size(),
-        2,
+        8,
         String.format(
             "Query: `%s` Results: %s",
             query,
@@ -1756,7 +1773,7 @@ public void testNegate() {
         String.format("%s - Expected search results to include matched fields", query));
     assertEquals(
         result.getEntities().size(),
-        2,
+        8,
         String.format(
             "Query: `%s` Results: %s",
             query,
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java
index e71865921678b..2ee96dd101da5 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java
@@ -102,5 +102,14 @@ public void testGetBrowsePath() throws Exception {
     List<String> browsePaths = browseDAO.getBrowsePaths(opContext, "dataset", dummyUrn);
     assertEquals(browsePaths.size(), 1);
     assertEquals(browsePaths.get(0), "foo");
+
+    // Test the case of null browsePaths field
+    sourceMap.put("browsePaths", Collections.singletonList(null));
+    when(mockSearchHit.getSourceAsMap()).thenReturn(sourceMap);
+    when(mockSearchHits.getHits()).thenReturn(new SearchHit[] {mockSearchHit});
+    when(mockSearchResponse.getHits()).thenReturn(mockSearchHits);
+    when(mockClient.search(any(), eq(RequestOptions.DEFAULT))).thenReturn(mockSearchResponse);
+    List<String> nullBrowsePaths = browseDAO.getBrowsePaths(opContext, "dataset", dummyUrn);
+    assertEquals(nullBrowsePaths.size(), 0);
   }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/BaseQueryFilterRewriterTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/BaseQueryFilterRewriterTest.java
new file mode 100644
index 0000000000000..6293d96fa35b8
--- /dev/null
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/BaseQueryFilterRewriterTest.java
@@ -0,0 +1,75 @@
+package com.linkedin.metadata.search.query.filter;
+
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+
+import com.linkedin.metadata.query.filter.Condition;
+import com.linkedin.metadata.search.elasticsearch.query.filter.BaseQueryFilterRewriter;
+import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriteChain;
+import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriterContext;
+import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriterSearchType;
+import io.datahubproject.metadata.context.OperationContext;
+import org.opensearch.index.query.BoolQueryBuilder;
+import org.opensearch.index.query.QueryBuilders;
+import org.testng.annotations.Test;
+
+public abstract class BaseQueryFilterRewriterTest<T extends BaseQueryFilterRewriter> {
+
+  abstract OperationContext getOpContext();
+
+  abstract T getTestRewriter();
+
+  abstract String getTargetField();
+
+  abstract String getTargetFieldValue();
+
+  abstract Condition getTargetCondition();
+
+  @Test
+  public void testPreservedMinimumMatchRewrite() {
+    BaseQueryFilterRewriter test = getTestRewriter();
+
+    // Setup nested container
+    BoolQueryBuilder testQuery = QueryBuilders.boolQuery().minimumShouldMatch(99);
+    testQuery.filter(
+        QueryBuilders.boolQuery()
+            .filter(
+                QueryBuilders.boolQuery()
+                    .filter(QueryBuilders.termsQuery(getTargetField(), getTargetFieldValue()))));
+    testQuery.filter(QueryBuilders.existsQuery("someField"));
+    testQuery.should(
+        QueryBuilders.boolQuery()
+            .minimumShouldMatch(100)
+            .should(
+                QueryBuilders.boolQuery()
+                    .minimumShouldMatch(101)
+                    .should(QueryBuilders.termsQuery(getTargetField(), getTargetFieldValue()))));
+
+    BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery().minimumShouldMatch(99);
+    expectedRewrite.filter(
+        QueryBuilders.boolQuery()
+            .filter(
+                QueryBuilders.boolQuery()
+                    .filter(QueryBuilders.termsQuery(getTargetField(), getTargetFieldValue()))));
+    expectedRewrite.filter(QueryBuilders.existsQuery("someField"));
+    expectedRewrite.should(
+        QueryBuilders.boolQuery()
+            .minimumShouldMatch(100)
+            .should(
+                QueryBuilders.boolQuery()
+                    .minimumShouldMatch(101)
+                    .should(QueryBuilders.termsQuery(getTargetField(), getTargetFieldValue()))));
+
+    assertEquals(
+        test.rewrite(
+            getOpContext(),
+            QueryFilterRewriterContext.builder()
+                .condition(getTargetCondition())
+                .searchType(QueryFilterRewriterSearchType.FULLTEXT_SEARCH)
+                .queryFilterRewriteChain(mock(QueryFilterRewriteChain.class))
+                .build(false),
+            testQuery),
+        expectedRewrite,
+        "Expected preservation of minimumShouldMatch");
+  }
+}
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java
index 5246e4dbe5bf9..1825b65a18ab1 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java
@@ -20,6 +20,7 @@
 import com.linkedin.metadata.aspect.models.graph.RelatedEntities;
 import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult;
 import com.linkedin.metadata.config.search.QueryFilterRewriterConfiguration;
+import com.linkedin.metadata.entity.SearchRetriever;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Condition;
@@ -39,7 +40,8 @@
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-public class ContainerExpansionRewriterTest {
+public class ContainerExpansionRewriterTest
+    extends BaseQueryFilterRewriterTest<ContainerExpansionRewriter> {
   private static final String FIELD_NAME = "container.keyword";
   private final String grandParentUrn = "urn:li:container:grand";
   private final String parentUrn = "urn:li:container:foo";
@@ -70,19 +72,46 @@ public void init() {
             () ->
                 io.datahubproject.metadata.context.RetrieverContext.builder()
                     .aspectRetriever(mockAspectRetriever)
+                    .cachingAspectRetriever(
+                        TestOperationContexts.emptyActiveUsersAspectRetriever(() -> entityRegistry))
                     .graphRetriever(mockGraphRetriever)
-                    .searchRetriever(TestOperationContexts.emptySearchRetriever)
+                    .searchRetriever(SearchRetriever.EMPTY)
                     .build(),
             null,
+            null,
             null);
   }
 
+  @Override
+  OperationContext getOpContext() {
+    return opContext;
+  }
+
+  @Override
+  ContainerExpansionRewriter getTestRewriter() {
+    return ContainerExpansionRewriter.builder()
+        .config(QueryFilterRewriterConfiguration.ExpansionRewriterConfiguration.DEFAULT)
+        .build();
+  }
+
+  @Override
+  String getTargetField() {
+    return FIELD_NAME;
+  }
+
+  @Override
+  String getTargetFieldValue() {
+    return childUrn;
+  }
+
+  @Override
+  Condition getTargetCondition() {
+    return Condition.ANCESTORS_INCL;
+  }
+
   @Test
   public void testTermsQueryRewrite() {
-    ContainerExpansionRewriter test =
-        ContainerExpansionRewriter.builder()
-            .config(QueryFilterRewriterConfiguration.ExpansionRewriterConfiguration.DEFAULT)
-            .build();
+    ContainerExpansionRewriter test = getTestRewriter();
 
     TermsQueryBuilder notTheFieldQuery = QueryBuilders.termsQuery("notTheField", childUrn);
     assertEquals(
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java
index edc6449438581..de375271ed660 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java
@@ -13,13 +13,14 @@
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 
-import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.aspect.RetrieverContext;
 import com.linkedin.metadata.aspect.models.graph.Edge;
 import com.linkedin.metadata.aspect.models.graph.RelatedEntities;
 import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult;
 import com.linkedin.metadata.config.search.QueryFilterRewriterConfiguration;
+import com.linkedin.metadata.entity.SearchRetriever;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Condition;
@@ -39,7 +40,8 @@
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-public class DomainExpansionRewriterTest {
+public class DomainExpansionRewriterTest
+    extends BaseQueryFilterRewriterTest<DomainExpansionRewriter> {
   private static final String FIELD_NAME = "domains.keyword";
   private final String grandParentUrn = "urn:li:domain:grand";
   private final String parentUrn = "urn:li:domain:foo";
@@ -53,7 +55,7 @@ public class DomainExpansionRewriterTest {
   @BeforeMethod
   public void init() {
     EntityRegistry entityRegistry = new TestEntityRegistry();
-    AspectRetriever mockAspectRetriever = mock(AspectRetriever.class);
+    CachingAspectRetriever mockAspectRetriever = mock(CachingAspectRetriever.class);
     when(mockAspectRetriever.getEntityRegistry()).thenReturn(entityRegistry);
 
     mockGraphRetriever = spy(GraphRetriever.class);
@@ -70,19 +72,46 @@ public void init() {
             () ->
                 io.datahubproject.metadata.context.RetrieverContext.builder()
                     .aspectRetriever(mockAspectRetriever)
+                    .cachingAspectRetriever(
+                        TestOperationContexts.emptyActiveUsersAspectRetriever(() -> entityRegistry))
                     .graphRetriever(mockGraphRetriever)
-                    .searchRetriever(TestOperationContexts.emptySearchRetriever)
+                    .searchRetriever(SearchRetriever.EMPTY)
                     .build(),
             null,
+            null,
             null);
   }
 
+  @Override
+  OperationContext getOpContext() {
+    return opContext;
+  }
+
+  @Override
+  DomainExpansionRewriter getTestRewriter() {
+    return DomainExpansionRewriter.builder()
+        .config(QueryFilterRewriterConfiguration.ExpansionRewriterConfiguration.DEFAULT)
+        .build();
+  }
+
+  @Override
+  String getTargetField() {
+    return FIELD_NAME;
+  }
+
+  @Override
+  String getTargetFieldValue() {
+    return parentUrn;
+  }
+
+  @Override
+  Condition getTargetCondition() {
+    return Condition.DESCENDANTS_INCL;
+  }
+
   @Test
   public void testTermsQueryRewrite() {
-    DomainExpansionRewriter test =
-        DomainExpansionRewriter.builder()
-            .config(QueryFilterRewriterConfiguration.ExpansionRewriterConfiguration.DEFAULT)
-            .build();
+    DomainExpansionRewriter test = getTestRewriter();
 
     TermsQueryBuilder notTheFieldQuery = QueryBuilders.termsQuery("notTheField", parentUrn);
     assertEquals(
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java
index 1381e9560b7e5..d6f5f9c3eedbe 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java
@@ -3,8 +3,7 @@
 import static com.linkedin.metadata.Constants.DATA_TYPE_URN_PREFIX;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
 import static com.linkedin.metadata.utils.SearchUtil.*;
-import static org.mockito.ArgumentMatchers.anySet;
-import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.*;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -12,23 +11,37 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.DataMap;
+import com.linkedin.data.template.LongMap;
 import com.linkedin.data.template.SetMode;
+import com.linkedin.data.template.StringArray;
 import com.linkedin.entity.Aspect;
 import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.config.search.SearchConfiguration;
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.models.annotation.SearchableAnnotation;
+import com.linkedin.metadata.query.filter.Condition;
+import com.linkedin.metadata.query.filter.Criterion;
+import com.linkedin.metadata.search.AggregationMetadata;
+import com.linkedin.metadata.search.FilterValue;
+import com.linkedin.metadata.search.FilterValueArray;
 import com.linkedin.metadata.search.elasticsearch.query.request.AggregationQueryBuilder;
 import com.linkedin.r2.RemoteInvocationException;
 import com.linkedin.structured.StructuredPropertyDefinition;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import org.mockito.Mockito;
 import org.opensearch.search.aggregations.AggregationBuilder;
 import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.testng.Assert;
@@ -37,16 +50,19 @@
 
 public class AggregationQueryBuilderTest {
 
-  private static AspectRetriever aspectRetriever;
-  private static AspectRetriever aspectRetrieverV1;
+  private static CachingAspectRetriever aspectRetriever;
+  private static CachingAspectRetriever aspectRetrieverV1;
+  private static String DEFAULT_FILTER = "_index";
 
   @BeforeClass
   public void setup() throws RemoteInvocationException, URISyntaxException {
     Urn helloUrn = Urn.createFromString("urn:li:structuredProperty:hello");
     Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten");
+    Urn underscoresAndDotsUrn =
+        Urn.createFromString("urn:li:structuredProperty:under.scores.and.dots_make_a_mess");
 
     // legacy
-    aspectRetriever = mock(AspectRetriever.class);
+    aspectRetriever = mock(CachingAspectRetriever.class);
     when(aspectRetriever.getEntityRegistry())
         .thenReturn(TestOperationContexts.defaultEntityRegistry());
 
@@ -75,8 +91,23 @@ public void setup() throws RemoteInvocationException, URISyntaxException {
                     STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
                     new Aspect(structPropAbFghTenDefinition.data()))));
 
+    StructuredPropertyDefinition structPropUnderscoresAndDotsDefinition =
+        new StructuredPropertyDefinition();
+    structPropUnderscoresAndDotsDefinition.setVersion(null, SetMode.REMOVE_IF_NULL);
+    structPropUnderscoresAndDotsDefinition.setValueType(
+        Urn.createFromString(DATA_TYPE_URN_PREFIX + "string"));
+    structPropUnderscoresAndDotsDefinition.setQualifiedName("under.scores.and.dots_make_a_mess");
+    structPropUnderscoresAndDotsDefinition.setDisplayName("under.scores.and.dots_make_a_mess");
+    when(aspectRetriever.getLatestAspectObjects(eq(Set.of(underscoresAndDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                underscoresAndDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(structPropUnderscoresAndDotsDefinition.data()))));
+
     // V1
-    aspectRetrieverV1 = mock(AspectRetriever.class);
+    aspectRetrieverV1 = mock(CachingAspectRetriever.class);
     when(aspectRetrieverV1.getEntityRegistry())
         .thenReturn(TestOperationContexts.defaultEntityRegistry());
 
@@ -105,6 +136,21 @@ public void setup() throws RemoteInvocationException, URISyntaxException {
                 Map.of(
                     STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
                     new Aspect(structPropAbFghTenDefinitionV1.data()))));
+
+    StructuredPropertyDefinition structPropUnderscoresAndDotsDefinitionV1 =
+        new StructuredPropertyDefinition();
+    structPropUnderscoresAndDotsDefinitionV1.setVersion("00000000000001");
+    structPropUnderscoresAndDotsDefinitionV1.setValueType(
+        Urn.createFromString(DATA_TYPE_URN_PREFIX + "string"));
+    structPropUnderscoresAndDotsDefinitionV1.setQualifiedName("under.scores.and.dots_make_a_mess");
+    structPropUnderscoresAndDotsDefinitionV1.setDisplayName("under.scores.and.dots_make_a_mess");
+    when(aspectRetrieverV1.getLatestAspectObjects(eq(Set.of(underscoresAndDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                underscoresAndDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(structPropUnderscoresAndDotsDefinitionV1.data()))));
   }
 
   @Test
@@ -221,16 +267,17 @@ public void testGetSpecificAggregationsHasFields() {
         builder.getAggregations(
             TestOperationContexts.systemContextNoSearchAuthorization(),
             ImmutableList.of("test1", "test2", "hasTest1"));
-    Assert.assertEquals(aggs.size(), 3);
+    Assert.assertEquals(aggs.size(), 4);
     Set<String> facets = aggs.stream().map(AggregationBuilder::getName).collect(Collectors.toSet());
-    Assert.assertEquals(ImmutableSet.of("test1", "test2", "hasTest1"), facets);
+    Assert.assertEquals(ImmutableSet.of("test1", "test2", "hasTest1", "_entityType"), facets);
 
     // Case 2: Ask for fields that should NOT exist.
     aggs =
         builder.getAggregations(
             TestOperationContexts.systemContextNoSearchAuthorization(),
             ImmutableList.of("hasTest2"));
-    Assert.assertEquals(aggs.size(), 0);
+    Assert.assertEquals(
+        aggs.size(), 1); // default has one field already, hasTest2 will not be in there
   }
 
   @Test
@@ -246,7 +293,7 @@ public void testAggregateOverStructuredProperty() {
         builder.getAggregations(
             TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever),
             List.of("structuredProperties.ab.fgh.ten"));
-    Assert.assertEquals(aggs.size(), 1);
+    Assert.assertEquals(aggs.size(), 2);
     AggregationBuilder aggBuilder = aggs.get(0);
     Assert.assertTrue(aggBuilder instanceof TermsAggregationBuilder);
     TermsAggregationBuilder agg = (TermsAggregationBuilder) aggBuilder;
@@ -261,12 +308,53 @@ public void testAggregateOverStructuredProperty() {
         builder.getAggregations(
             TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever),
             List.of("structuredProperties.ab.fgh.ten", "structuredProperties.hello"));
+    Assert.assertEquals(
+        aggs.size(), 3); // has one default filter (_entityType) both get mapped to _index
+    Assert.assertEquals(
+        aggs.stream()
+            .map(aggr -> ((TermsAggregationBuilder) aggr).field())
+            .collect(Collectors.toSet()),
+        Set.of(
+            "structuredProperties.ab_fgh_ten.keyword",
+            "structuredProperties.hello.keyword",
+            DEFAULT_FILTER));
+  }
+
+  @Test
+  public void testAggregateOverStructuredPropertyNamespaced() {
+    SearchConfiguration config = new SearchConfiguration();
+    config.setMaxTermBucketSize(25);
+
+    AggregationQueryBuilder builder =
+        new AggregationQueryBuilder(
+            config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of()));
+
+    List<AggregationBuilder> aggs =
+        builder.getAggregations(
+            TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever),
+            List.of("structuredProperties.under.scores.and.dots_make_a_mess"));
     Assert.assertEquals(aggs.size(), 2);
     Assert.assertEquals(
         aggs.stream()
             .map(aggr -> ((TermsAggregationBuilder) aggr).field())
             .collect(Collectors.toSet()),
-        Set.of("structuredProperties.ab_fgh_ten.keyword", "structuredProperties.hello.keyword"));
+        Set.of("structuredProperties.under_scores_and_dots_make_a_mess.keyword", DEFAULT_FILTER));
+    // Two structured properties
+    aggs =
+        builder.getAggregations(
+            TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever),
+            List.of(
+                "structuredProperties.under.scores.and.dots_make_a_mess",
+                "structuredProperties.hello"));
+    Assert.assertEquals(aggs.size(), 3);
+    Assert.assertEquals(
+        aggs.stream()
+            .map(aggr -> ((TermsAggregationBuilder) aggr).field())
+            .collect(Collectors.toSet()),
+        Set.of(
+            "structuredProperties.under_scores_and_dots_make_a_mess.keyword",
+            "structuredProperties.hello.keyword",
+            DEFAULT_FILTER));
   }
 
   @Test
@@ -282,7 +370,7 @@ public void testAggregateOverStructuredPropertyV1() {
         builder.getAggregations(
             TestOperationContexts.systemContextNoSearchAuthorization(aspectRetrieverV1),
             List.of("structuredProperties.ab.fgh.ten"));
-    Assert.assertEquals(aggs.size(), 1);
+    Assert.assertEquals(aggs.size(), 2);
     AggregationBuilder aggBuilder = aggs.get(0);
     Assert.assertTrue(aggBuilder instanceof TermsAggregationBuilder);
     TermsAggregationBuilder agg = (TermsAggregationBuilder) aggBuilder;
@@ -299,14 +387,56 @@ public void testAggregateOverStructuredPropertyV1() {
             List.of(
                 "structuredProperties.ab.fgh.ten",
                 "structuredProperties._versioned.hello.00000000000001.string"));
-    Assert.assertEquals(aggs.size(), 2);
+    Assert.assertEquals(
+        aggs.size(), 3); // has two one filter (_entityType) both get mapped to _index
     Assert.assertEquals(
         aggs.stream()
             .map(aggr -> ((TermsAggregationBuilder) aggr).field())
             .collect(Collectors.toSet()),
         Set.of(
             "structuredProperties._versioned.ab_fgh_ten.00000000000001.string.keyword",
-            "structuredProperties._versioned.hello.00000000000001.string.keyword"));
+            "structuredProperties._versioned.hello.00000000000001.string.keyword",
+            DEFAULT_FILTER));
+  }
+
+  @Test
+  public void testAggregateOverStructuredPropertyNamespacedV1() {
+    SearchConfiguration config = new SearchConfiguration();
+    config.setMaxTermBucketSize(25);
+
+    AggregationQueryBuilder builder =
+        new AggregationQueryBuilder(
+            config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of()));
+
+    List<AggregationBuilder> aggs =
+        builder.getAggregations(
+            TestOperationContexts.systemContextNoSearchAuthorization(aspectRetrieverV1),
+            List.of("structuredProperties.under.scores.and.dots_make_a_mess"));
+    Assert.assertEquals(aggs.size(), 2);
+    Assert.assertEquals(
+        aggs.stream()
+            .map(aggr -> ((TermsAggregationBuilder) aggr).field())
+            .collect(Collectors.toSet()),
+        Set.of(
+            "structuredProperties._versioned.under_scores_and_dots_make_a_mess.00000000000001.string.keyword",
+            DEFAULT_FILTER));
+
+    // Two structured properties
+    aggs =
+        builder.getAggregations(
+            TestOperationContexts.systemContextNoSearchAuthorization(aspectRetrieverV1),
+            List.of(
+                "structuredProperties.under.scores.and.dots_make_a_mess",
+                "structuredProperties._versioned.hello.00000000000001.string"));
+    Assert.assertEquals(aggs.size(), 3);
+    Assert.assertEquals(
+        aggs.stream()
+            .map(aggr -> ((TermsAggregationBuilder) aggr).field())
+            .collect(Collectors.toSet()),
+        Set.of(
+            "structuredProperties._versioned.under_scores_and_dots_make_a_mess.00000000000001.string.keyword",
+            "structuredProperties._versioned.hello.00000000000001.string.keyword",
+            DEFAULT_FILTER));
   }
 
   @Test
@@ -363,7 +493,7 @@ public void testAggregateOverFieldsAndStructProp() {
                 "hasTest1",
                 "structuredProperties.ab.fgh.ten",
                 "structuredProperties.hello"));
-    Assert.assertEquals(aggs.size(), 5);
+    Assert.assertEquals(aggs.size(), 6);
     Set<String> facets =
         aggs.stream()
             .map(aggB -> ((TermsAggregationBuilder) aggB).field())
@@ -375,7 +505,8 @@ public void testAggregateOverFieldsAndStructProp() {
             "test2.keyword",
             "hasTest1",
             "structuredProperties.ab_fgh_ten.keyword",
-            "structuredProperties.hello.keyword"));
+            "structuredProperties.hello.keyword",
+            DEFAULT_FILTER));
   }
 
   @Test
@@ -432,7 +563,8 @@ public void testAggregateOverFieldsAndStructPropV1() {
                 "hasTest1",
                 "structuredProperties.ab.fgh.ten",
                 "structuredProperties.hello"));
-    Assert.assertEquals(aggs.size(), 5);
+    Assert.assertEquals(
+        aggs.size(), 6); // has one default filter (_entityType) both get mapped to _index
     Set<String> facets =
         aggs.stream()
             .map(aggB -> ((TermsAggregationBuilder) aggB).field())
@@ -444,7 +576,8 @@ public void testAggregateOverFieldsAndStructPropV1() {
             "test2.keyword",
             "hasTest1",
             "structuredProperties._versioned.ab_fgh_ten.00000000000001.string.keyword",
-            "structuredProperties._versioned.hello.00000000000001.string.keyword"));
+            "structuredProperties._versioned.hello.00000000000001.string.keyword",
+            DEFAULT_FILTER));
   }
 
   @Test
@@ -486,4 +619,129 @@ public void testMissingAggregation() {
                         .equals(
                             MISSING_SPECIAL_TYPE + AGGREGATION_SPECIAL_TYPE_DELIMITER + "test")));
   }
+
+  @Test
+  public void testUpdateAggregationEntityWithStructuredProp() {
+    final AggregationMetadata aggregationMetadata = new AggregationMetadata();
+    aggregationMetadata.setName("structuredProperties.test_me.one");
+
+    SearchConfiguration config = new SearchConfiguration();
+    config.setMaxTermBucketSize(25);
+
+    AggregationQueryBuilder builder =
+        new AggregationQueryBuilder(
+            config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of()));
+
+    builder.updateAggregationEntity(aggregationMetadata);
+    Assert.assertEquals(
+        aggregationMetadata.getEntity(), UrnUtils.getUrn("urn:li:structuredProperty:test_me.one"));
+  }
+
+  @Test
+  public void testUpdateAggregationEntityWithRegularFilter() {
+    final AggregationMetadata aggregationMetadata = new AggregationMetadata();
+    aggregationMetadata.setName("domains");
+
+    SearchConfiguration config = new SearchConfiguration();
+    config.setMaxTermBucketSize(25);
+
+    AggregationQueryBuilder builder =
+        new AggregationQueryBuilder(
+            config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of()));
+
+    builder.updateAggregationEntity(aggregationMetadata);
+    Assert.assertNull(aggregationMetadata.getEntity());
+  }
+
+  @Test
+  public void testAddFiltersToMetadataWithStructuredPropsNoResults() {
+    final Urn propertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:test_me.one");
+
+    SearchConfiguration config = new SearchConfiguration();
+    config.setMaxTermBucketSize(25);
+
+    AggregationQueryBuilder builder =
+        new AggregationQueryBuilder(
+            config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of()));
+
+    Criterion criterion =
+        new Criterion()
+            .setField("structuredProperties.test_me.one")
+            .setValues(new StringArray("test123"))
+            .setCondition(Condition.EQUAL);
+
+    AspectRetriever mockAspectRetriever = getMockAspectRetriever(propertyUrn);
+
+    final List<AggregationMetadata> aggregationMetadataList = new ArrayList<>();
+    builder.addCriterionFiltersToAggregationMetadata(
+        criterion, aggregationMetadataList, mockAspectRetriever);
+
+    // ensure we add the correct structured prop aggregation here
+    Assert.assertEquals(aggregationMetadataList.size(), 1);
+    Assert.assertEquals(aggregationMetadataList.get(0).getEntity(), propertyUrn);
+    Assert.assertEquals(
+        aggregationMetadataList.get(0).getName(), "structuredProperties.test_me.one");
+    Assert.assertEquals(aggregationMetadataList.get(0).getAggregations().size(), 1);
+    Assert.assertEquals(aggregationMetadataList.get(0).getAggregations().get("test123"), 0);
+  }
+
+  @Test
+  public void testAddFiltersToMetadataWithStructuredPropsWithAggregations() {
+    final Urn propertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:test_me.one");
+
+    final AggregationMetadata aggregationMetadata = new AggregationMetadata();
+    aggregationMetadata.setName("structuredProperties.test_me.one");
+    aggregationMetadata.setEntity(propertyUrn);
+    FilterValue filterValue =
+        new FilterValue().setValue("test123").setFiltered(false).setFacetCount(1);
+    aggregationMetadata.setFilterValues(new FilterValueArray(filterValue));
+    LongMap aggregations = new LongMap();
+    aggregations.put("test123", 1L);
+    aggregationMetadata.setAggregations(aggregations);
+
+    SearchConfiguration config = new SearchConfiguration();
+    config.setMaxTermBucketSize(25);
+
+    AggregationQueryBuilder builder =
+        new AggregationQueryBuilder(
+            config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of()));
+
+    Criterion criterion =
+        new Criterion()
+            .setField("structuredProperties.test_me.one")
+            .setValues(new StringArray("test123"))
+            .setCondition(Condition.EQUAL);
+
+    AspectRetriever mockAspectRetriever = getMockAspectRetriever(propertyUrn);
+
+    final List<AggregationMetadata> aggregationMetadataList = new ArrayList<>();
+    aggregationMetadataList.add(aggregationMetadata);
+    builder.addCriterionFiltersToAggregationMetadata(
+        criterion, aggregationMetadataList, mockAspectRetriever);
+
+    Assert.assertEquals(aggregationMetadataList.size(), 1);
+    Assert.assertEquals(aggregationMetadataList.get(0).getEntity(), propertyUrn);
+    Assert.assertEquals(
+        aggregationMetadataList.get(0).getName(), "structuredProperties.test_me.one");
+    Assert.assertEquals(aggregationMetadataList.get(0).getAggregations().size(), 1);
+    Assert.assertEquals(aggregationMetadataList.get(0).getAggregations().get("test123"), 1);
+  }
+
+  private AspectRetriever getMockAspectRetriever(Urn propertyUrn) {
+    AspectRetriever mockAspectRetriever = Mockito.mock(AspectRetriever.class);
+    Map<Urn, Map<String, Aspect>> mockResult = new HashMap<>();
+    Map<String, Aspect> aspectMap = new HashMap<>();
+    DataMap definition = new DataMap();
+    definition.put("qualifiedName", "test_me.one");
+    definition.put("valueType", "urn:li:dataType:datahub.string");
+    Aspect definitionAspect = new Aspect(definition);
+    aspectMap.put(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, definitionAspect);
+    mockResult.put(propertyUrn, aspectMap);
+    Set<Urn> urns = new HashSet<>();
+    urns.add(propertyUrn);
+    Mockito.when(mockAspectRetriever.getLatestAspectObjects(eq(urns), any()))
+        .thenReturn(mockResult);
+
+    return mockAspectRetriever;
+  }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java
index 572d79ebf2f0c..c5205906e9d37 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java
@@ -5,6 +5,10 @@
 import static org.testng.Assert.assertTrue;
 
 import com.linkedin.metadata.TestEntitySpecBuilder;
+import com.linkedin.metadata.config.search.ExactMatchConfiguration;
+import com.linkedin.metadata.config.search.PartialConfiguration;
+import com.linkedin.metadata.config.search.SearchConfiguration;
+import com.linkedin.metadata.config.search.WordGramConfiguration;
 import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration;
 import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration;
 import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration;
@@ -32,14 +36,44 @@
 import org.testng.annotations.Test;
 
 public class AutocompleteRequestHandlerTest {
-  private AutocompleteRequestHandler handler =
-      AutocompleteRequestHandler.getBuilder(
-          TestEntitySpecBuilder.getSpec(),
-          CustomSearchConfiguration.builder().build(),
-          QueryFilterRewriteChain.EMPTY);
+  private static SearchConfiguration testQueryConfig;
+  private static AutocompleteRequestHandler handler;
   private OperationContext mockOpContext =
       TestOperationContexts.systemContextNoSearchAuthorization(mock(EntityRegistry.class));
 
+  static {
+    testQueryConfig = new SearchConfiguration();
+    testQueryConfig.setMaxTermBucketSize(20);
+
+    ExactMatchConfiguration exactMatchConfiguration = new ExactMatchConfiguration();
+    exactMatchConfiguration.setExclusive(false);
+    exactMatchConfiguration.setExactFactor(10.0f);
+    exactMatchConfiguration.setWithPrefix(true);
+    exactMatchConfiguration.setPrefixFactor(6.0f);
+    exactMatchConfiguration.setCaseSensitivityFactor(0.7f);
+    exactMatchConfiguration.setEnableStructured(true);
+
+    WordGramConfiguration wordGramConfiguration = new WordGramConfiguration();
+    wordGramConfiguration.setTwoGramFactor(1.2f);
+    wordGramConfiguration.setThreeGramFactor(1.5f);
+    wordGramConfiguration.setFourGramFactor(1.8f);
+
+    PartialConfiguration partialConfiguration = new PartialConfiguration();
+    partialConfiguration.setFactor(0.4f);
+    partialConfiguration.setUrnFactor(0.7f);
+
+    testQueryConfig.setExactMatch(exactMatchConfiguration);
+    testQueryConfig.setWordGram(wordGramConfiguration);
+    testQueryConfig.setPartial(partialConfiguration);
+
+    handler =
+        AutocompleteRequestHandler.getBuilder(
+            TestEntitySpecBuilder.getSpec(),
+            CustomSearchConfiguration.builder().build(),
+            QueryFilterRewriteChain.EMPTY,
+            testQueryConfig);
+  }
+
   private static final QueryConfiguration TEST_QUERY_CONFIG =
       QueryConfiguration.builder()
           .queryRegex(".*")
@@ -88,9 +122,12 @@ public void testDefaultAutocompleteRequest() {
     BoolQueryBuilder wrapper =
         (BoolQueryBuilder) ((FunctionScoreQueryBuilder) sourceBuilder.query()).query();
     BoolQueryBuilder query = (BoolQueryBuilder) extractNestedQuery(wrapper);
-    assertEquals(query.should().size(), 3);
+    assertEquals(query.should().size(), 4);
 
-    MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(2);
+    MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) query.should().get(0);
+    assertEquals("keyPart1.keyword", matchQueryBuilder.fieldName());
+
+    MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(3);
     Map<String, Float> queryFields = autocompleteQuery.fields();
     assertTrue(queryFields.containsKey("keyPart1.ngram"));
     assertTrue(queryFields.containsKey("keyPart1.ngram._2gram"));
@@ -99,7 +136,7 @@ public void testDefaultAutocompleteRequest() {
     assertEquals(autocompleteQuery.type(), MultiMatchQueryBuilder.Type.BOOL_PREFIX);
 
     MatchPhrasePrefixQueryBuilder prefixQuery =
-        (MatchPhrasePrefixQueryBuilder) query.should().get(0);
+        (MatchPhrasePrefixQueryBuilder) query.should().get(1);
     assertEquals("keyPart1.delimited", prefixQuery.fieldName());
 
     assertEquals(wrapper.mustNot().size(), 1);
@@ -108,15 +145,16 @@ public void testDefaultAutocompleteRequest() {
     assertEquals(removedFilter.value(), true);
     HighlightBuilder highlightBuilder = sourceBuilder.highlighter();
     List<HighlightBuilder.Field> highlightedFields = highlightBuilder.fields();
-    assertEquals(highlightedFields.size(), 8);
+    assertEquals(highlightedFields.size(), 9);
     assertEquals(highlightedFields.get(0).name(), "keyPart1");
     assertEquals(highlightedFields.get(1).name(), "keyPart1.*");
     assertEquals(highlightedFields.get(2).name(), "keyPart1.ngram");
     assertEquals(highlightedFields.get(3).name(), "keyPart1.delimited");
-    assertEquals(highlightedFields.get(4).name(), "urn");
-    assertEquals(highlightedFields.get(5).name(), "urn.*");
-    assertEquals(highlightedFields.get(6).name(), "urn.ngram");
-    assertEquals(highlightedFields.get(7).name(), "urn.delimited");
+    assertEquals(highlightedFields.get(4).name(), "keyPart1.keyword");
+    assertEquals(highlightedFields.get(5).name(), "urn");
+    assertEquals(highlightedFields.get(6).name(), "urn.*");
+    assertEquals(highlightedFields.get(7).name(), "urn.ngram");
+    assertEquals(highlightedFields.get(8).name(), "urn.delimited");
   }
 
   @Test
@@ -130,9 +168,12 @@ public void testAutocompleteRequestWithField() {
         (BoolQueryBuilder) ((FunctionScoreQueryBuilder) sourceBuilder.query()).query();
     assertEquals(wrapper.should().size(), 1);
     BoolQueryBuilder query = (BoolQueryBuilder) extractNestedQuery(wrapper);
-    assertEquals(query.should().size(), 2);
+    assertEquals(query.should().size(), 3);
 
-    MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(1);
+    MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) query.should().get(0);
+    assertEquals("field.keyword", matchQueryBuilder.fieldName());
+
+    MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(2);
     Map<String, Float> queryFields = autocompleteQuery.fields();
     assertTrue(queryFields.containsKey("field.ngram"));
     assertTrue(queryFields.containsKey("field.ngram._2gram"));
@@ -141,7 +182,7 @@ public void testAutocompleteRequestWithField() {
     assertEquals(autocompleteQuery.type(), MultiMatchQueryBuilder.Type.BOOL_PREFIX);
 
     MatchPhrasePrefixQueryBuilder prefixQuery =
-        (MatchPhrasePrefixQueryBuilder) query.should().get(0);
+        (MatchPhrasePrefixQueryBuilder) query.should().get(1);
     assertEquals("field.delimited", prefixQuery.fieldName());
 
     MatchQueryBuilder removedFilter = (MatchQueryBuilder) wrapper.mustNot().get(0);
@@ -149,11 +190,12 @@ public void testAutocompleteRequestWithField() {
     assertEquals(removedFilter.value(), true);
     HighlightBuilder highlightBuilder = sourceBuilder.highlighter();
     List<HighlightBuilder.Field> highlightedFields = highlightBuilder.fields();
-    assertEquals(highlightedFields.size(), 4);
+    assertEquals(highlightedFields.size(), 5);
     assertEquals(highlightedFields.get(0).name(), "field");
     assertEquals(highlightedFields.get(1).name(), "field.*");
     assertEquals(highlightedFields.get(2).name(), "field.ngram");
     assertEquals(highlightedFields.get(3).name(), "field.delimited");
+    assertEquals(highlightedFields.get(4).name(), "field.keyword");
   }
 
   @Test
@@ -174,7 +216,8 @@ public void testCustomConfigWithDefault() {
                                     .build())
                             .build()))
                 .build(),
-            QueryFilterRewriteChain.EMPTY);
+            QueryFilterRewriteChain.EMPTY,
+            testQueryConfig);
 
     SearchRequest autocompleteRequest =
         withoutDefaultQuery.getSearchRequest(mockOpContext, "input", null, null, 10);
@@ -200,7 +243,8 @@ public void testCustomConfigWithDefault() {
                                     .build())
                             .build()))
                 .build(),
-            QueryFilterRewriteChain.EMPTY);
+            QueryFilterRewriteChain.EMPTY,
+            testQueryConfig);
 
     autocompleteRequest = withDefaultQuery.getSearchRequest(mockOpContext, "input", null, null, 10);
     sourceBuilder = autocompleteRequest.source();
@@ -215,7 +259,7 @@ public void testCustomConfigWithDefault() {
     BoolQueryBuilder defaultQuery =
         (BoolQueryBuilder)
             shouldQueries.stream().filter(qb -> qb instanceof BoolQueryBuilder).findFirst().get();
-    assertEquals(defaultQuery.should().size(), 3);
+    assertEquals(defaultQuery.should().size(), 4);
 
     // Custom
     customQuery =
@@ -243,7 +287,8 @@ public void testCustomConfigWithInheritedQueryFunctionScores() {
                                     .build())
                             .build()))
                 .build(),
-            QueryFilterRewriteChain.EMPTY);
+            QueryFilterRewriteChain.EMPTY,
+            testQueryConfig);
 
     SearchRequest autocompleteRequest =
         withInherit.getSearchRequest(mockOpContext, "input", null, null, 10);
@@ -282,7 +327,8 @@ public void testCustomConfigWithInheritedQueryFunctionScores() {
                                     .build())
                             .build()))
                 .build(),
-            QueryFilterRewriteChain.EMPTY);
+            QueryFilterRewriteChain.EMPTY,
+            testQueryConfig);
 
     autocompleteRequest =
         noQueryCustomization.getSearchRequest(mockOpContext, "input", null, null, 10);
@@ -345,7 +391,8 @@ public void testCustomConfigWithFunctionScores() {
                                                     "deprecated", Map.of("value", false)))))))
                             .build()))
                 .build(),
-            QueryFilterRewriteChain.EMPTY);
+            QueryFilterRewriteChain.EMPTY,
+            testQueryConfig);
 
     SearchRequest autocompleteRequest =
         explicitNoInherit.getSearchRequest(mockOpContext, "input", null, null, 10);
@@ -398,7 +445,8 @@ public void testCustomConfigWithFunctionScores() {
                                                     "deprecated", Map.of("value", false)))))))
                             .build()))
                 .build(),
-            QueryFilterRewriteChain.EMPTY);
+            QueryFilterRewriteChain.EMPTY,
+            testQueryConfig);
 
     autocompleteRequest = explicit.getSearchRequest(mockOpContext, "input", null, null, 10);
     sourceBuilder = autocompleteRequest.source();
@@ -411,7 +459,7 @@ public void testCustomConfigWithFunctionScores() {
     assertEquals(customQuery, QueryBuilders.matchAllQuery());
 
     // standard query still present
-    assertEquals(((BoolQueryBuilder) query.should().get(1)).should().size(), 3);
+    assertEquals(((BoolQueryBuilder) query.should().get(1)).should().size(), 4);
 
     // custom functions included
     assertEquals(wrapper.filterFunctionBuilders(), expectedCustomScoreFunctions);
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
index a3ef62760d797..e51511699e345 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
@@ -1,24 +1,34 @@
 package com.linkedin.metadata.search.query.request;
 
 import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES;
+import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME;
 import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion;
 import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion;
 import static com.linkedin.metadata.utils.CriterionUtils.buildIsNullCriterion;
 import static com.linkedin.metadata.utils.SearchUtil.*;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 import static org.testng.Assert.*;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.data.DataMap;
 import com.linkedin.data.template.StringArray;
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
+import com.linkedin.entity.Aspect;
 import com.linkedin.metadata.TestEntitySpecBuilder;
+import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.config.search.ExactMatchConfiguration;
 import com.linkedin.metadata.config.search.PartialConfiguration;
 import com.linkedin.metadata.config.search.SearchConfiguration;
 import com.linkedin.metadata.config.search.WordGramConfiguration;
+import com.linkedin.metadata.entity.SearchRetriever;
 import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
@@ -28,6 +38,8 @@
 import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriteChain;
 import com.linkedin.metadata.search.elasticsearch.query.request.SearchRequestHandler;
 import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.metadata.context.RetrieverContext;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
 import io.datahubproject.test.search.config.SearchCommonTestConfiguration;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -286,7 +298,8 @@ public void testAggregationsInSearch() {
         String.format("_entityType%stextFieldOverride", AGGREGATION_SEPARATOR_CHAR);
     SearchRequest searchRequest =
         requestHandler.getSearchRequest(
-            operationContext.withSearchFlags(flags -> flags.setFulltext(true)),
+            operationContext.withSearchFlags(
+                flags -> flags.setFulltext(true).setIncludeDefaultFacets(false)),
             "*",
             null,
             null,
@@ -633,6 +646,50 @@ public void testBrowsePathQueryFilter() {
     assertEquals(((ExistsQueryBuilder) mustHaveV1.must().get(0)).fieldName(), "browsePaths");
   }
 
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testInvalidStructuredProperty() {
+    AspectRetriever aspectRetriever = mock(AspectRetriever.class);
+    Map<Urn, Map<String, Aspect>> aspectResponse = new HashMap<>();
+    DataMap statusData = new DataMap();
+    statusData.put("removed", true);
+    Aspect status = new Aspect(statusData);
+    Urn structPropUrn = StructuredPropertyUtils.toURNFromFQN("under.scores.and.dots.make_a_mess");
+    aspectResponse.put(structPropUrn, ImmutableMap.of(STATUS_ASPECT_NAME, status));
+    when(aspectRetriever.getLatestAspectObjects(
+            Collections.singleton(structPropUrn), ImmutableSet.of(STATUS_ASPECT_NAME)))
+        .thenReturn(aspectResponse);
+    OperationContext mockRetrieverContext =
+        TestOperationContexts.systemContextNoSearchAuthorization(
+            RetrieverContext.builder()
+                .aspectRetriever(aspectRetriever)
+                .cachingAspectRetriever(TestOperationContexts.emptyActiveUsersAspectRetriever(null))
+                .graphRetriever(mock(GraphRetriever.class))
+                .searchRetriever(mock(SearchRetriever.class))
+                .build());
+
+    Criterion structuredPropCriterion =
+        buildExistsCriterion("structuredProperties.under.scores.and.dots.make_a_mess");
+
+    CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(structuredPropCriterion);
+
+    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
+    conjunctiveCriterion.setAnd(criterionArray);
+
+    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
+    conjunctiveCriterionArray.add(conjunctiveCriterion);
+
+    Filter filter = new Filter();
+    filter.setOr(conjunctiveCriterionArray);
+
+    BoolQueryBuilder test =
+        SearchRequestHandler.getFilterQuery(
+            mockRetrieverContext.withSearchFlags(flags -> flags.setFulltext(false)),
+            filter,
+            new HashMap<>(),
+            QueryFilterRewriteChain.EMPTY);
+  }
+
   @Test
   public void testQueryByDefault() {
     final Set<String> COMMON =
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java
index 2c5bcd1294fa1..65b73b7425b74 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java
@@ -247,6 +247,9 @@ public void testSetSearchableRefValue() throws URISyntaxException, RemoteInvocat
         TestOperationContexts.systemContextNoSearchAuthorization(
             RetrieverContext.builder()
                 .aspectRetriever(aspectRetriever)
+                .cachingAspectRetriever(
+                    TestOperationContexts.emptyActiveUsersAspectRetriever(
+                        () -> TEST_ENTITY_REGISTRY))
                 .graphRetriever(mock(GraphRetriever.class))
                 .searchRetriever(mock(SearchRetriever.class))
                 .build());
@@ -301,6 +304,9 @@ public void testSetSearchableRefValue_RuntimeException()
         TestOperationContexts.systemContextNoSearchAuthorization(
             RetrieverContext.builder()
                 .aspectRetriever(aspectRetriever)
+                .cachingAspectRetriever(
+                    TestOperationContexts.emptyActiveUsersAspectRetriever(
+                        () -> TEST_ENTITY_REGISTRY))
                 .graphRetriever(mock(GraphRetriever.class))
                 .searchRetriever(mock(SearchRetriever.class))
                 .build());
@@ -337,6 +343,9 @@ public void testSetSearchableRefValue_RuntimeException_URNExist()
         TestOperationContexts.systemContextNoSearchAuthorization(
             RetrieverContext.builder()
                 .aspectRetriever(aspectRetriever)
+                .cachingAspectRetriever(
+                    TestOperationContexts.emptyActiveUsersAspectRetriever(
+                        () -> TEST_ENTITY_REGISTRY))
                 .graphRetriever(mock(GraphRetriever.class))
                 .searchRetriever(mock(SearchRetriever.class))
                 .build());
@@ -369,6 +378,9 @@ void testSetSearchableRefValue_WithInvalidURN()
         TestOperationContexts.systemContextNoSearchAuthorization(
             RetrieverContext.builder()
                 .aspectRetriever(aspectRetriever)
+                .cachingAspectRetriever(
+                    TestOperationContexts.emptyActiveUsersAspectRetriever(
+                        () -> TEST_ENTITY_REGISTRY))
                 .graphRetriever(mock(GraphRetriever.class))
                 .searchRetriever(mock(SearchRetriever.class))
                 .build());
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESAccessControlUtilTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESAccessControlUtilTest.java
index b3dcaf174ed38..12eb468fcc61a 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESAccessControlUtilTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESAccessControlUtilTest.java
@@ -97,6 +97,7 @@ public class ESAccessControlUtilTest {
           null,
           null,
           null,
+          null,
           null);
 
   private static final String VIEW_PRIVILEGE = "VIEW_ENTITY_PAGE";
@@ -106,6 +107,18 @@ public class ESAccessControlUtilTest {
   private static final Urn RESTRICTED_RESULT_URN =
       UrnUtils.getUrn("urn:li:restricted:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)");
 
+  private static final String PREFIX_MATCH =
+      "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.human";
+  private static final Urn PREFIX_MATCH_URN =
+      UrnUtils.getUrn(
+          "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.humans,PROD)");
+  private static final Urn PREFIX_NO_MATCH_URN =
+      UrnUtils.getUrn(
+          "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.meta_humans,PROD)");
+  private static final Urn RESTRICTED_PREFIX_NO_MATCH_URN =
+      UrnUtils.getUrn(
+          "urn:li:restricted:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.meta_humans,PROD)");
+
   /** Comprehensive list of policy variations */
   private static final Map<String, DataHubPolicyInfo> TEST_POLICIES =
       ImmutableMap.<String, DataHubPolicyInfo>builder()
@@ -251,6 +264,30 @@ public class ESAccessControlUtilTest {
                                                   .setValues(
                                                       new StringArray(
                                                           List.of(DOMAIN_A.toString())))))))))
+          .put(
+              "urnPrefixAllUsers",
+              new DataHubPolicyInfo()
+                  .setDisplayName("")
+                  .setState(PoliciesConfig.ACTIVE_POLICY_STATE)
+                  .setType(PoliciesConfig.METADATA_POLICY_TYPE)
+                  .setActors(
+                      new DataHubActorFilter()
+                          .setAllUsers(true)
+                          .setGroups(new UrnArray())
+                          .setUsers(new UrnArray()))
+                  .setPrivileges(new StringArray(List.of(VIEW_PRIVILEGE)))
+                  .setResources(
+                      new DataHubResourceFilter()
+                          .setFilter(
+                              new PolicyMatchFilter()
+                                  .setCriteria(
+                                      new PolicyMatchCriterionArray(
+                                          List.of(
+                                              new PolicyMatchCriterion()
+                                                  .setField("URN")
+                                                  .setCondition(PolicyMatchCondition.STARTS_WITH)
+                                                  .setValues(
+                                                      new StringArray(List.of(PREFIX_MATCH)))))))))
           .build();
 
   /** User A is a technical owner of the result User B has no ownership */
@@ -264,7 +301,7 @@ public void testAllUsersRestrictions() throws RemoteInvocationException, URISynt
 
     // USER A
     OperationContext userAContext =
-        sessionWithUserAGroupAandC(Set.of(TEST_POLICIES.get("allUsers")));
+        sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("allUsers")));
 
     SearchResult result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -272,20 +309,18 @@ public void testAllUsersRestrictions() throws RemoteInvocationException, URISynt
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     result = mockSearchResult();
-    ESAccessControlUtil.restrictSearchResult(
-        userAContext.withSearchFlags(flags -> flags.setIncludeRestricted(false)), result);
+    ESAccessControlUtil.restrictSearchResult(userAContext, result);
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     // USER B
-    OperationContext userBContext = sessionWithUserBNoGroup(Set.of(TEST_POLICIES.get("allUsers")));
+    OperationContext userBContext = sessionWithUserBNoGroup(List.of(TEST_POLICIES.get("allUsers")));
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
         userBContext.withSearchFlags(flags -> flags.setIncludeRestricted(true)), result);
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     result = mockSearchResult();
-    ESAccessControlUtil.restrictSearchResult(
-        userBContext.withSearchFlags(flags -> flags.setIncludeRestricted(false)), result);
+    ESAccessControlUtil.restrictSearchResult(userBContext, result);
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
   }
 
@@ -293,7 +328,7 @@ public void testAllUsersRestrictions() throws RemoteInvocationException, URISynt
   public void testSingeUserRestrictions() throws RemoteInvocationException, URISyntaxException {
 
     // USER A
-    OperationContext userAContext = sessionWithUserAGroupAandC(Set.of(TEST_POLICIES.get("userA")));
+    OperationContext userAContext = sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("userA")));
 
     SearchResult result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -301,12 +336,11 @@ public void testSingeUserRestrictions() throws RemoteInvocationException, URISyn
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     result = mockSearchResult();
-    ESAccessControlUtil.restrictSearchResult(
-        userAContext.withSearchFlags(flags -> flags.setIncludeRestricted(false)), result);
+    ESAccessControlUtil.restrictSearchResult(userAContext, result);
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     // USER B
-    OperationContext userBContext = sessionWithUserBNoGroup(Set.of(TEST_POLICIES.get("userA")));
+    OperationContext userBContext = sessionWithUserBNoGroup(List.of(TEST_POLICIES.get("userA")));
 
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -322,7 +356,7 @@ public void testAllGroupsRestrictions() throws RemoteInvocationException, URISyn
 
     // USER A
     OperationContext userAContext =
-        sessionWithUserAGroupAandC(Set.of(TEST_POLICIES.get("allGroups")));
+        sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("allGroups")));
 
     SearchResult result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -330,12 +364,12 @@ public void testAllGroupsRestrictions() throws RemoteInvocationException, URISyn
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     result = mockSearchResult();
-    ESAccessControlUtil.restrictSearchResult(
-        userAContext.withSearchFlags(flags -> flags.setIncludeRestricted(false)), result);
+    ESAccessControlUtil.restrictSearchResult(userAContext, result);
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     // USER B (No Groups!)
-    OperationContext userBContext = sessionWithUserBNoGroup(Set.of(TEST_POLICIES.get("allGroups")));
+    OperationContext userBContext =
+        sessionWithUserBNoGroup(List.of(TEST_POLICIES.get("allGroups")));
 
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -352,7 +386,7 @@ public void testSingleGroupRestrictions() throws RemoteInvocationException, URIS
     // GROUP B Policy
     // USER A
     final OperationContext userAContext =
-        sessionWithUserAGroupAandC(Set.of(TEST_POLICIES.get("groupB")));
+        sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("groupB")));
 
     SearchResult result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -364,7 +398,7 @@ public void testSingleGroupRestrictions() throws RemoteInvocationException, URIS
 
     // USER B (No Groups!)
     final OperationContext userBContext =
-        sessionWithUserBNoGroup(Set.of(TEST_POLICIES.get("groupB")));
+        sessionWithUserBNoGroup(List.of(TEST_POLICIES.get("groupB")));
 
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -377,7 +411,7 @@ public void testSingleGroupRestrictions() throws RemoteInvocationException, URIS
     // Group C Policy
     // USER A
     final OperationContext userAGroupCContext =
-        sessionWithUserAGroupAandC(Set.of(TEST_POLICIES.get("groupC")));
+        sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("groupC")));
 
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -385,13 +419,12 @@ public void testSingleGroupRestrictions() throws RemoteInvocationException, URIS
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     result = mockSearchResult();
-    ESAccessControlUtil.restrictSearchResult(
-        userAGroupCContext.withSearchFlags(flags -> flags.setIncludeRestricted(false)), result);
+    ESAccessControlUtil.restrictSearchResult(userAGroupCContext, result);
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     // USER B (No Groups!)
     final OperationContext userBgroupCContext =
-        sessionWithUserBNoGroup(Set.of(TEST_POLICIES.get("groupC")));
+        sessionWithUserBNoGroup(List.of(TEST_POLICIES.get("groupC")));
 
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -407,7 +440,7 @@ public void testAnyOwnerRestrictions() throws RemoteInvocationException, URISynt
 
     // USER A
     OperationContext userAContext =
-        sessionWithUserAGroupAandC(Set.of(TEST_POLICIES.get("anyOwner")));
+        sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("anyOwner")));
 
     SearchResult result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -415,12 +448,11 @@ public void testAnyOwnerRestrictions() throws RemoteInvocationException, URISynt
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     result = mockSearchResult();
-    ESAccessControlUtil.restrictSearchResult(
-        userAContext.withSearchFlags(flags -> flags.setIncludeRestricted(false)), result);
+    ESAccessControlUtil.restrictSearchResult(userAContext, result);
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     // USER B (not an owner)
-    OperationContext userBContext = sessionWithUserBNoGroup(Set.of(TEST_POLICIES.get("anyOwner")));
+    OperationContext userBContext = sessionWithUserBNoGroup(List.of(TEST_POLICIES.get("anyOwner")));
 
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -436,7 +468,7 @@ public void testBusinessOwnerRestrictions() throws RemoteInvocationException, UR
 
     // USER A
     final OperationContext userAContext =
-        sessionWithUserAGroupAandC(Set.of(TEST_POLICIES.get("businessOwner")));
+        sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("businessOwner")));
 
     SearchResult result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -448,7 +480,7 @@ public void testBusinessOwnerRestrictions() throws RemoteInvocationException, UR
 
     // USER B
     final OperationContext userBContext =
-        sessionWithUserBNoGroup(Set.of(TEST_POLICIES.get("businessOwner")));
+        sessionWithUserBNoGroup(List.of(TEST_POLICIES.get("businessOwner")));
 
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -464,7 +496,7 @@ public void testDomainRestrictions() throws RemoteInvocationException, URISyntax
 
     // USER A
     OperationContext userAContext =
-        sessionWithUserAGroupAandC(Set.of(TEST_POLICIES.get("domainA")));
+        sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("domainA")));
 
     SearchResult result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
@@ -477,7 +509,7 @@ public void testDomainRestrictions() throws RemoteInvocationException, URISyntax
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
 
     // USER B
-    OperationContext userBContext = sessionWithUserBNoGroup(Set.of(TEST_POLICIES.get("domainA")));
+    OperationContext userBContext = sessionWithUserBNoGroup(List.of(TEST_POLICIES.get("domainA")));
     result = mockSearchResult();
     ESAccessControlUtil.restrictSearchResult(
         userBContext.withSearchFlags(flags -> flags.setIncludeRestricted(true)), result);
@@ -489,6 +521,28 @@ public void testDomainRestrictions() throws RemoteInvocationException, URISyntax
     assertEquals(result.getEntities().get(0).getEntity(), UNRESTRICTED_RESULT_URN);
   }
 
+  @Test
+  public void testPrefixRestrictions() throws RemoteInvocationException, URISyntaxException {
+
+    // USER A
+    OperationContext userAContext =
+        sessionWithUserAGroupAandC(List.of(TEST_POLICIES.get("urnPrefixAllUsers")));
+
+    SearchResult result = mockPrefixSearchResult();
+    ESAccessControlUtil.restrictSearchResult(
+        userAContext.withSearchFlags(flags -> flags.setIncludeRestricted(true)), result);
+    assertEquals(result.getEntities().size(), 2);
+    assertEquals(result.getEntities().get(0).getEntity(), PREFIX_MATCH_URN);
+    assertEquals(result.getEntities().get(1).getEntity(), RESTRICTED_PREFIX_NO_MATCH_URN);
+
+    result = mockPrefixSearchResult();
+    ESAccessControlUtil.restrictSearchResult(
+        userAContext.withSearchFlags(flags -> flags.setIncludeRestricted(false)), result);
+    assertEquals(result.getEntities().size(), 2);
+    assertEquals(result.getEntities().get(0).getEntity(), PREFIX_MATCH_URN);
+    assertEquals(result.getEntities().get(1).getEntity(), PREFIX_NO_MATCH_URN);
+  }
+
   private static RestrictedService mockRestrictedService() {
     RestrictedService mockRestrictedService = mock(RestrictedService.class);
     when(mockRestrictedService.encryptRestrictedUrn(any()))
@@ -500,6 +554,23 @@ private static RestrictedService mockRestrictedService() {
     return mockRestrictedService;
   }
 
+  private static SearchResult mockPrefixSearchResult() {
+    SearchResult result = new SearchResult();
+    result.setFrom(0);
+    result.setPageSize(10);
+    result.setNumEntities(1);
+    result.setEntities(
+        new SearchEntityArray(
+            new SearchEntity()
+                .setEntity(PREFIX_MATCH_URN)
+                .setMatchedFields(new MatchedFieldArray()),
+            new SearchEntity()
+                .setEntity(PREFIX_NO_MATCH_URN)
+                .setMatchedFields(new MatchedFieldArray())));
+    result.setMetadata(mock(SearchResultMetadata.class));
+    return result;
+  }
+
   private static SearchResult mockSearchResult() {
     SearchResult result = new SearchResult();
     result.setFrom(0);
@@ -514,18 +585,18 @@ private static SearchResult mockSearchResult() {
     return result;
   }
 
-  private static OperationContext sessionWithUserAGroupAandC(Set<DataHubPolicyInfo> policies)
+  private static OperationContext sessionWithUserAGroupAandC(List<DataHubPolicyInfo> policies)
       throws RemoteInvocationException, URISyntaxException {
     return sessionWithUserGroups(USER_A_AUTH, policies, List.of(TEST_GROUP_A, TEST_GROUP_C));
   }
 
-  private static OperationContext sessionWithUserBNoGroup(Set<DataHubPolicyInfo> policies)
+  private static OperationContext sessionWithUserBNoGroup(List<DataHubPolicyInfo> policies)
       throws RemoteInvocationException, URISyntaxException {
     return sessionWithUserGroups(USER_B_AUTH, policies, List.of());
   }
 
   private static OperationContext sessionWithUserGroups(
-      Authentication auth, Set<DataHubPolicyInfo> policies, List<Urn> groups)
+      Authentication auth, List<DataHubPolicyInfo> policies, List<Urn> groups)
       throws RemoteInvocationException, URISyntaxException {
     Urn actorUrn = UrnUtils.getUrn(auth.getActor().toUrnStr());
     Authorizer dataHubAuthorizer =
@@ -538,7 +609,7 @@ public static class TestDataHubAuthorizer extends DataHubAuthorizer {
 
     public TestDataHubAuthorizer(
         @Nonnull OperationContext opContext,
-        @Nonnull Set<DataHubPolicyInfo> policies,
+        @Nonnull List<DataHubPolicyInfo> policies,
         @Nonnull Map<Urn, List<Urn>> userGroups,
         @Nonnull Map<Urn, Map<Urn, Set<Urn>>> resourceOwnerTypes)
         throws RemoteInvocationException, URISyntaxException {
@@ -569,6 +640,7 @@ public TestDataHubAuthorizer(
                     Collectors.groupingBy(
                         Pair::getKey, Collectors.mapping(Pair::getValue, Collectors.toList())));
         policyCache.putAll(byPrivilegeName);
+        policyCache.put(ALL, policies);
       } finally {
         readWriteLock.writeLock().unlock();
       }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java
index 6665faacae337..4f2bda39ad211 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java
@@ -43,6 +43,9 @@ public class ESUtilsTest {
   @BeforeClass
   public static void setup() throws RemoteInvocationException, URISyntaxException {
     Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten");
+    Urn underscoresAndDotsUrn =
+        Urn.createFromString("urn:li:structuredProperty:under.scores.and.dots_make_a_mess");
+    Urn dateWithDotsUrn = Urn.createFromString("urn:li:structuredProperty:date_here.with_dot");
 
     // legacy
     aspectRetriever = mock(AspectRetriever.class);
@@ -62,6 +65,32 @@ public static void setup() throws RemoteInvocationException, URISyntaxException
                     STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
                     new Aspect(structPropAbFghTenDefinition.data()))));
 
+    StructuredPropertyDefinition dateWithDotsDefinition = new StructuredPropertyDefinition();
+    dateWithDotsDefinition.setVersion(null, SetMode.REMOVE_IF_NULL);
+    dateWithDotsDefinition.setValueType(Urn.createFromString(DATA_TYPE_URN_PREFIX + "date"));
+    dateWithDotsDefinition.setQualifiedName("date_here.with_dot");
+    when(aspectRetriever.getLatestAspectObjects(eq(Set.of(dateWithDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                dateWithDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(dateWithDotsDefinition.data()))));
+
+    StructuredPropertyDefinition structPropUnderscoresAndDotsDefinition =
+        new StructuredPropertyDefinition();
+    structPropUnderscoresAndDotsDefinition.setVersion(null, SetMode.REMOVE_IF_NULL);
+    structPropUnderscoresAndDotsDefinition.setValueType(
+        Urn.createFromString(DATA_TYPE_URN_PREFIX + "string"));
+    structPropUnderscoresAndDotsDefinition.setQualifiedName("under.scores.and.dots_make_a_mess");
+    when(aspectRetriever.getLatestAspectObjects(eq(Set.of(underscoresAndDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                underscoresAndDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(structPropUnderscoresAndDotsDefinition.data()))));
+
     // V1
     aspectRetrieverV1 = mock(AspectRetriever.class);
     when(aspectRetrieverV1.getEntityRegistry())
@@ -80,6 +109,20 @@ public static void setup() throws RemoteInvocationException, URISyntaxException
                 Map.of(
                     STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
                     new Aspect(structPropAbFghTenDefinitionV1.data()))));
+
+    StructuredPropertyDefinition structPropUnderscoresAndDotsDefinitionV1 =
+        new StructuredPropertyDefinition();
+    structPropUnderscoresAndDotsDefinitionV1.setVersion("00000000000001");
+    structPropUnderscoresAndDotsDefinitionV1.setValueType(
+        Urn.createFromString(DATA_TYPE_URN_PREFIX + "string"));
+    structPropUnderscoresAndDotsDefinitionV1.setQualifiedName("under.scores.and.dots_make_a_mess");
+    when(aspectRetrieverV1.getLatestAspectObjects(eq(Set.of(underscoresAndDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                underscoresAndDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(structPropUnderscoresAndDotsDefinitionV1.data()))));
   }
 
   @Test
@@ -677,6 +720,86 @@ public void testGetQueryBuilderFromCriterionFieldToExpand() {
             + "  }\n"
             + "}";
     Assert.assertEquals(result.toString(), expected);
+
+    final Criterion originCriterion = buildCriterion("origin", Condition.EQUAL, "PROD");
+
+    // Ensure that the query is expanded!
+    QueryBuilder originExpanded =
+        ESUtils.getQueryBuilderFromCriterion(
+            originCriterion,
+            false,
+            new HashMap<>(),
+            mock(OperationContext.class),
+            QueryFilterRewriteChain.EMPTY);
+    String originExpected =
+        "{\n"
+            + "  \"bool\" : {\n"
+            + "    \"should\" : [\n"
+            + "      {\n"
+            + "        \"terms\" : {\n"
+            + "          \"origin.keyword\" : [\n"
+            + "            \"PROD\"\n"
+            + "          ],\n"
+            + "          \"boost\" : 1.0,\n"
+            + "          \"_name\" : \"origin\"\n"
+            + "        }\n"
+            + "      },\n"
+            + "      {\n"
+            + "        \"terms\" : {\n"
+            + "          \"env.keyword\" : [\n"
+            + "            \"PROD\"\n"
+            + "          ],\n"
+            + "          \"boost\" : 1.0,\n"
+            + "          \"_name\" : \"env\"\n"
+            + "        }\n"
+            + "      }\n"
+            + "    ],\n"
+            + "    \"adjust_pure_negative\" : true,\n"
+            + "    \"minimum_should_match\" : \"1\",\n"
+            + "    \"boost\" : 1.0\n"
+            + "  }\n"
+            + "}";
+    Assert.assertEquals(originExpanded.toString(), originExpected);
+
+    final Criterion envCriterion = buildCriterion("env", Condition.EQUAL, "PROD");
+
+    // Ensure that the query is expanded!
+    QueryBuilder envExpanded =
+        ESUtils.getQueryBuilderFromCriterion(
+            envCriterion,
+            false,
+            new HashMap<>(),
+            mock(OperationContext.class),
+            QueryFilterRewriteChain.EMPTY);
+    String envExpected =
+        "{\n"
+            + "  \"bool\" : {\n"
+            + "    \"should\" : [\n"
+            + "      {\n"
+            + "        \"terms\" : {\n"
+            + "          \"env.keyword\" : [\n"
+            + "            \"PROD\"\n"
+            + "          ],\n"
+            + "          \"boost\" : 1.0,\n"
+            + "          \"_name\" : \"env\"\n"
+            + "        }\n"
+            + "      },\n"
+            + "      {\n"
+            + "        \"terms\" : {\n"
+            + "          \"origin.keyword\" : [\n"
+            + "            \"PROD\"\n"
+            + "          ],\n"
+            + "          \"boost\" : 1.0,\n"
+            + "          \"_name\" : \"origin\"\n"
+            + "        }\n"
+            + "      }\n"
+            + "    ],\n"
+            + "    \"adjust_pure_negative\" : true,\n"
+            + "    \"minimum_should_match\" : \"1\",\n"
+            + "    \"boost\" : 1.0\n"
+            + "  }\n"
+            + "}";
+    Assert.assertEquals(envExpanded.toString(), envExpected);
   }
 
   @Test
@@ -703,6 +826,31 @@ public void testGetQueryBuilderFromStructPropEqualsValue() {
     Assert.assertEquals(result.toString(), expected);
   }
 
+  @Test
+  public void testGetQueryBuilderFromNamespacedStructPropEqualsValue() {
+
+    final Criterion singleValueCriterion =
+        buildCriterion(
+            "structuredProperties.under.scores.and.dots_make_a_mess", Condition.EQUAL, "value1");
+
+    OperationContext opContext = mock(OperationContext.class);
+    when(opContext.getAspectRetriever()).thenReturn(aspectRetriever);
+    QueryBuilder result =
+        ESUtils.getQueryBuilderFromCriterion(
+            singleValueCriterion, false, new HashMap<>(), opContext, QueryFilterRewriteChain.EMPTY);
+    String expected =
+        "{\n"
+            + "  \"terms\" : {\n"
+            + "    \"structuredProperties.under_scores_and_dots_make_a_mess.keyword\" : [\n"
+            + "      \"value1\"\n"
+            + "    ],\n"
+            + "    \"boost\" : 1.0,\n"
+            + "    \"_name\" : \"structuredProperties.under.scores.and.dots_make_a_mess\"\n"
+            + "  }\n"
+            + "}";
+    Assert.assertEquals(result.toString(), expected);
+  }
+
   @Test
   public void testGetQueryBuilderFromStructPropEqualsValueV1() {
 
@@ -731,6 +879,65 @@ public void testGetQueryBuilderFromStructPropEqualsValueV1() {
     Assert.assertEquals(result.toString(), expected);
   }
 
+  @Test
+  public void testGetQueryBuilderFromNamespacedStructPropEqualsValueV1() {
+
+    final Criterion singleValueCriterion =
+        buildCriterion(
+            "structuredProperties.under.scores.and.dots_make_a_mess", Condition.EQUAL, "value1");
+
+    OperationContext opContextV1 = mock(OperationContext.class);
+    when(opContextV1.getAspectRetriever()).thenReturn(aspectRetrieverV1);
+    QueryBuilder result =
+        ESUtils.getQueryBuilderFromCriterion(
+            singleValueCriterion,
+            false,
+            new HashMap<>(),
+            opContextV1,
+            QueryFilterRewriteChain.EMPTY);
+    String expected =
+        "{\n"
+            + "  \"terms\" : {\n"
+            + "    \"structuredProperties._versioned.under_scores_and_dots_make_a_mess.00000000000001.string.keyword\" : [\n"
+            + "      \"value1\"\n"
+            + "    ],\n"
+            + "    \"boost\" : 1.0,\n"
+            + "    \"_name\" : \"structuredProperties.under.scores.and.dots_make_a_mess\"\n"
+            + "  }\n"
+            + "}";
+    Assert.assertEquals(result.toString(), expected);
+  }
+
+  @Test
+  public void testGetQueryBuilderFromDatesWithDots() {
+
+    final Criterion singleValueCriterion =
+        buildCriterion(
+            "structuredProperties.date_here.with_dot", Condition.GREATER_THAN, "1731974400000");
+
+    OperationContext opContext = mock(OperationContext.class);
+    when(opContext.getAspectRetriever()).thenReturn(aspectRetriever);
+    QueryBuilder result =
+        ESUtils.getQueryBuilderFromCriterion(
+            singleValueCriterion, false, new HashMap<>(), opContext, QueryFilterRewriteChain.EMPTY);
+    // structuredProperties.date_here_with_dot should not have .keyword at the end since this field
+    // type is type long for dates
+    String expected =
+        "{\n"
+            + "  \"range\" : {\n"
+            + "    \"structuredProperties.date_here_with_dot\" : {\n"
+            + "      \"from\" : 1731974400000,\n"
+            + "      \"to\" : null,\n"
+            + "      \"include_lower\" : false,\n"
+            + "      \"include_upper\" : true,\n"
+            + "      \"boost\" : 1.0,\n"
+            + "      \"_name\" : \"structuredProperties.date_here.with_dot\"\n"
+            + "    }\n"
+            + "  }\n"
+            + "}";
+    Assert.assertEquals(result.toString(), expected);
+  }
+
   @Test
   public void testGetQueryBuilderFromStructPropExists() {
     final Criterion singleValueCriterion = buildExistsCriterion("structuredProperties.ab.fgh.ten");
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateGraphIndicesServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateGraphIndicesServiceTest.java
index 03e381a9059da..dd02b1fdc9d78 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateGraphIndicesServiceTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateGraphIndicesServiceTest.java
@@ -1,6 +1,11 @@
 package com.linkedin.metadata.service;
 
+import static com.linkedin.metadata.Constants.CONTAINER_ENTITY_NAME;
+import static com.linkedin.metadata.search.utils.QueryUtils.createRelationshipFilter;
+import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.times;
@@ -8,9 +13,11 @@
 import static org.mockito.Mockito.verifyNoInteractions;
 import static org.testng.Assert.assertEquals;
 
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.Status;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.container.Container;
 import com.linkedin.dataset.DatasetProperties;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.Constants;
@@ -21,6 +28,14 @@
 import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.models.registry.LineageRegistry;
+import com.linkedin.metadata.query.filter.Condition;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.Criterion;
+import com.linkedin.metadata.query.filter.CriterionArray;
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.query.filter.RelationshipDirection;
+import com.linkedin.metadata.query.filter.RelationshipFilter;
 import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder;
 import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor;
 import com.linkedin.metadata.utils.GenericRecordUtils;
@@ -29,6 +44,8 @@
 import com.linkedin.mxe.MetadataChangeLog;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
+import java.util.List;
+import javax.annotation.Nonnull;
 import org.mockito.ArgumentCaptor;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.script.Script;
@@ -180,4 +197,109 @@ public void testStatusNoOpEvent() {
 
     verifyNoInteractions(mockWriteDAO);
   }
+
+  @Test
+  public void testMissingAspectGraphDelete() {
+    // Test deleting a null aspect
+    test.handleChangeEvent(
+        TEST_OP_CONTEXT,
+        new MetadataChangeLog()
+            .setChangeType(ChangeType.DELETE)
+            .setEntityType(TEST_URN.getEntityType())
+            .setEntityUrn(TEST_URN)
+            .setAspectName(Constants.CONTAINER_ASPECT_NAME));
+
+    // For missing aspects, verify no writes
+    verifyNoInteractions(mockWriteDAO);
+  }
+
+  @Test
+  public void testNodeGraphDelete() {
+    Urn containerUrn = UrnUtils.getUrn("urn:li:container:foo");
+
+    // Test deleting container entity
+    test.handleChangeEvent(
+        TEST_OP_CONTEXT,
+        new MetadataChangeLog()
+            .setChangeType(ChangeType.DELETE)
+            .setEntityType(CONTAINER_ENTITY_NAME)
+            .setEntityUrn(containerUrn)
+            .setAspectName(Constants.CONTAINER_KEY_ASPECT_NAME));
+
+    // Delete all outgoing edges of this entity
+    verify(mockWriteDAO, times(1))
+        .deleteByQuery(
+            eq(TEST_OP_CONTEXT),
+            nullable(String.class),
+            eq(createUrnFilter(containerUrn)),
+            nullable(String.class),
+            eq(new Filter().setOr(new ConjunctiveCriterionArray())),
+            eq(List.of()),
+            eq(new RelationshipFilter().setDirection(RelationshipDirection.OUTGOING)));
+
+    // Delete all incoming edges of this entity
+    verify(mockWriteDAO, times(1))
+        .deleteByQuery(
+            eq(TEST_OP_CONTEXT),
+            nullable(String.class),
+            eq(createUrnFilter(containerUrn)),
+            nullable(String.class),
+            eq(new Filter().setOr(new ConjunctiveCriterionArray())),
+            eq(List.of()),
+            eq(new RelationshipFilter().setDirection(RelationshipDirection.INCOMING)));
+
+    // Delete all edges where this entity is a lifecycle owner
+    verify(mockWriteDAO, times(1))
+        .deleteByQuery(
+            eq(TEST_OP_CONTEXT),
+            nullable(String.class),
+            eq(new Filter().setOr(new ConjunctiveCriterionArray())),
+            nullable(String.class),
+            eq(new Filter().setOr(new ConjunctiveCriterionArray())),
+            eq(List.of()),
+            eq(new RelationshipFilter().setDirection(RelationshipDirection.INCOMING)),
+            eq(containerUrn.toString()));
+  }
+
+  @Test
+  public void testContainerDelete() {
+    Urn containerUrn = UrnUtils.getUrn("urn:li:container:foo");
+
+    // Test deleting a container aspect
+    test.handleChangeEvent(
+        TEST_OP_CONTEXT,
+        new MetadataChangeLog()
+            .setChangeType(ChangeType.DELETE)
+            .setEntityType(TEST_URN.getEntityType())
+            .setEntityUrn(TEST_URN)
+            .setAspectName(Constants.CONTAINER_ASPECT_NAME)
+            .setPreviousAspectValue(
+                GenericRecordUtils.serializeAspect(new Container().setContainer(containerUrn))));
+
+    // For container aspects, verify that only edges are removed in both cases
+    verify(mockWriteDAO, times(1))
+        .deleteByQuery(
+            eq(TEST_OP_CONTEXT),
+            nullable(String.class),
+            eq(createUrnFilter(TEST_URN)),
+            nullable(String.class),
+            eq(new Filter().setOr(new ConjunctiveCriterionArray())),
+            eq(List.of("IsPartOf")),
+            eq(
+                createRelationshipFilter(
+                    new Filter().setOr(new ConjunctiveCriterionArray()),
+                    RelationshipDirection.OUTGOING)));
+  }
+
+  private static Filter createUrnFilter(@Nonnull final Urn urn) {
+    Filter filter = new Filter();
+    CriterionArray criterionArray = new CriterionArray();
+    Criterion criterion = buildCriterion("urn", Condition.EQUAL, urn.toString());
+    criterionArray.add(criterion);
+    filter.setOr(
+        new ConjunctiveCriterionArray(
+            ImmutableList.of(new ConjunctiveCriterion().setAnd(criterionArray))));
+
+    return filter;
+  }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java
new file mode 100644
index 0000000000000..43f8cc0ef191d
--- /dev/null
+++ b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java
@@ -0,0 +1,84 @@
+package com.linkedin.metadata.service;
+
+import static com.linkedin.metadata.Constants.CONTAINER_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.verify;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.template.RecordTemplate;
+import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.models.AspectSpec;
+import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.metadata.search.EntitySearchService;
+import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders;
+import com.linkedin.metadata.search.transformer.SearchDocumentTransformer;
+import com.linkedin.metadata.systemmetadata.SystemMetadataService;
+import com.linkedin.metadata.timeseries.TimeseriesAspectService;
+import com.linkedin.metadata.utils.SystemMetadataUtils;
+import com.linkedin.mxe.MetadataChangeLog;
+import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class UpdateIndicesServiceTest {
+
+  @Mock private UpdateGraphIndicesService updateGraphIndicesService;
+  @Mock private EntitySearchService entitySearchService;
+  @Mock private TimeseriesAspectService timeseriesAspectService;
+  @Mock private SystemMetadataService systemMetadataService;
+  @Mock private SearchDocumentTransformer searchDocumentTransformer;
+  @Mock private EntityIndexBuilders entityIndexBuilders;
+
+  private OperationContext operationContext;
+  private UpdateIndicesService updateIndicesService;
+
+  @BeforeMethod
+  public void setup() {
+    MockitoAnnotations.openMocks(this);
+    operationContext = TestOperationContexts.systemContextNoSearchAuthorization();
+    updateIndicesService =
+        new UpdateIndicesService(
+            updateGraphIndicesService,
+            entitySearchService,
+            timeseriesAspectService,
+            systemMetadataService,
+            searchDocumentTransformer,
+            entityIndexBuilders,
+            "MD5");
+  }
+
+  @Test
+  public void testContainerHandleDeleteEvent() throws Exception {
+    Urn urn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)");
+    EntitySpec entitySpec = operationContext.getEntityRegistry().getEntitySpec(DATASET_ENTITY_NAME);
+    AspectSpec aspectSpec = entitySpec.getAspectSpec(CONTAINER_ASPECT_NAME);
+
+    // Create test data
+    MetadataChangeLog event = new MetadataChangeLog();
+    event.setChangeType(ChangeType.DELETE);
+    event.setEntityUrn(urn);
+    event.setAspectName(CONTAINER_ASPECT_NAME);
+    event.setEntityType(urn.getEntityType());
+    event.setSystemMetadata(SystemMetadataUtils.createDefaultSystemMetadata());
+
+    // Execute Delete
+    updateIndicesService.handleChangeEvent(operationContext, event);
+
+    // Verify
+    verify(systemMetadataService).deleteAspect(urn.toString(), CONTAINER_ASPECT_NAME);
+    verify(searchDocumentTransformer)
+        .transformAspect(
+            eq(operationContext),
+            eq(urn),
+            nullable(RecordTemplate.class),
+            eq(aspectSpec),
+            eq(true));
+    verify(updateGraphIndicesService).handleChangeEvent(operationContext, event);
+  }
+}
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffectTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffectTest.java
index b1b716c560481..9a0a82c7f9f49 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffectTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffectTest.java
@@ -18,7 +18,8 @@
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.entity.Aspect;
 import com.linkedin.events.metadata.ChangeType;
-import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
+import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.batch.PatchMCP;
 import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
@@ -36,7 +37,6 @@
 import com.linkedin.test.metadata.aspect.TestEntityRegistry;
 import com.linkedin.test.metadata.aspect.batch.TestMCL;
 import io.datahubproject.metadata.context.RetrieverContext;
-import io.datahubproject.test.metadata.context.TestOperationContexts;
 import jakarta.json.Json;
 import jakarta.json.JsonPatch;
 import java.util.List;
@@ -76,13 +76,13 @@ public class PropertyDefinitionDeleteSideEffectTest {
   private static final Urn TEST_DATASET_URN =
       UrnUtils.getUrn(
           "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)");
-  private AspectRetriever mockAspectRetriever;
+  private CachingAspectRetriever mockAspectRetriever;
   private SearchRetriever mockSearchRetriever;
   private RetrieverContext retrieverContext;
 
   @BeforeMethod
   public void setup() {
-    mockAspectRetriever = mock(AspectRetriever.class);
+    mockAspectRetriever = mock(CachingAspectRetriever.class);
     when(mockAspectRetriever.getEntityRegistry()).thenReturn(TEST_REGISTRY);
     when(mockAspectRetriever.getLatestAspectObject(
             eq(TEST_PROPERTY_URN), eq(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)))
@@ -101,8 +101,8 @@ public void setup() {
     retrieverContext =
         RetrieverContext.builder()
             .searchRetriever(mockSearchRetriever)
-            .aspectRetriever(mockAspectRetriever)
-            .graphRetriever(TestOperationContexts.emptyGraphRetriever)
+            .cachingAspectRetriever(mockAspectRetriever)
+            .graphRetriever(GraphRetriever.EMPTY)
             .build();
   }
 
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/HidePropertyValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/HidePropertyValidatorTest.java
new file mode 100644
index 0000000000000..3f664da0dde8a
--- /dev/null
+++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/HidePropertyValidatorTest.java
@@ -0,0 +1,55 @@
+package com.linkedin.metadata.structuredproperties.validators;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import com.linkedin.metadata.structuredproperties.validation.HidePropertyValidator;
+import com.linkedin.structured.StructuredPropertySettings;
+import com.linkedin.test.metadata.aspect.TestEntityRegistry;
+import com.linkedin.test.metadata.aspect.batch.TestMCP;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class HidePropertyValidatorTest {
+
+  private static final EntityRegistry TEST_REGISTRY = new TestEntityRegistry();
+
+  private static final Urn TEST_PROPERTY_URN =
+      UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime");
+
+  @Test
+  public void testValidUpsert() {
+
+    StructuredPropertySettings propertySettings =
+        new StructuredPropertySettings()
+            .setIsHidden(false)
+            .setShowAsAssetBadge(true)
+            .setShowInAssetSummary(true)
+            .setShowInSearchFilters(true);
+
+    boolean isValid =
+        HidePropertyValidator.validateSettingsUpserts(
+                TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY))
+            .findAny()
+            .isEmpty();
+    Assert.assertTrue(isValid);
+  }
+
+  @Test
+  public void testInvalidUpsert() {
+
+    StructuredPropertySettings propertySettings =
+        new StructuredPropertySettings()
+            .setIsHidden(true)
+            .setShowAsAssetBadge(true)
+            .setShowInAssetSummary(true)
+            .setShowInSearchFilters(true);
+
+    boolean isValid =
+        HidePropertyValidator.validateSettingsUpserts(
+                TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY))
+            .findAny()
+            .isEmpty();
+    Assert.assertFalse(isValid);
+  }
+}
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java
index 2af731a51145e..18949f0566dd1 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java
@@ -397,4 +397,40 @@ public void testCanChangeAllowedValueDescriptions()
             .count(),
         0);
   }
+
+  @Test
+  public void testUrnIdWithSpace()
+      throws URISyntaxException, CloneNotSupportedException, AspectValidationException {
+    Urn propertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:test me out.foo.bar");
+    StructuredPropertyDefinition newProperty = new StructuredPropertyDefinition();
+    newProperty.setEntityTypes(new UrnArray(Urn.createFromString("urn:li:logicalEntity:dataset")));
+    newProperty.setDisplayName("oldProp");
+    newProperty.setQualifiedName("foo.bar");
+    newProperty.setCardinality(PropertyCardinality.MULTIPLE);
+    newProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING"));
+    assertEquals(
+        PropertyDefinitionValidator.validateDefinitionUpserts(
+                TestMCP.ofOneMCP(propertyUrn, null, newProperty, entityRegistry),
+                mockRetrieverContext)
+            .count(),
+        1);
+  }
+
+  @Test
+  public void testQualifiedNameWithSpace()
+      throws URISyntaxException, CloneNotSupportedException, AspectValidationException {
+    Urn propertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:foo.bar");
+    StructuredPropertyDefinition newProperty = new StructuredPropertyDefinition();
+    newProperty.setEntityTypes(new UrnArray(Urn.createFromString("urn:li:logicalEntity:dataset")));
+    newProperty.setDisplayName("oldProp");
+    newProperty.setQualifiedName("foo.bar with spaces");
+    newProperty.setCardinality(PropertyCardinality.MULTIPLE);
+    newProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING"));
+    assertEquals(
+        PropertyDefinitionValidator.validateDefinitionUpserts(
+                TestMCP.ofOneMCP(propertyUrn, null, newProperty, entityRegistry),
+                mockRetrieverContext)
+            .count(),
+        1);
+  }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/ShowPropertyAsBadgeValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/ShowPropertyAsBadgeValidatorTest.java
new file mode 100644
index 0000000000000..6e8886f495c95
--- /dev/null
+++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/ShowPropertyAsBadgeValidatorTest.java
@@ -0,0 +1,160 @@
+package com.linkedin.metadata.structuredproperties.validators;
+
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.metadata.aspect.GraphRetriever;
+import com.linkedin.metadata.aspect.RetrieverContext;
+import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import com.linkedin.metadata.entity.SearchRetriever;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.search.ScrollResult;
+import com.linkedin.metadata.search.SearchEntity;
+import com.linkedin.metadata.search.SearchEntityArray;
+import com.linkedin.metadata.structuredproperties.validation.ShowPropertyAsBadgeValidator;
+import com.linkedin.structured.StructuredPropertySettings;
+import com.linkedin.test.metadata.aspect.MockAspectRetriever;
+import com.linkedin.test.metadata.aspect.TestEntityRegistry;
+import com.linkedin.test.metadata.aspect.batch.TestMCP;
+import java.util.Collections;
+import java.util.stream.Stream;
+import org.mockito.Mockito;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class ShowPropertyAsBadgeValidatorTest {
+
+  private static final EntityRegistry TEST_REGISTRY = new TestEntityRegistry();
+  private static final Urn TEST_PROPERTY_URN =
+      UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime");
+  private static final Urn EXISTING_BADGE_URN =
+      UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.existingBadge");
+
+  private SearchRetriever mockSearchRetriever;
+  private MockAspectRetriever mockAspectRetriever;
+  private GraphRetriever mockGraphRetriever;
+  private RetrieverContext retrieverContext;
+
+  @BeforeMethod
+  public void setup() {
+    mockSearchRetriever = Mockito.mock(SearchRetriever.class);
+
+    StructuredPropertySettings propertySettings =
+        new StructuredPropertySettings()
+            .setShowAsAssetBadge(true)
+            .setShowInAssetSummary(true)
+            .setShowInSearchFilters(true);
+    mockAspectRetriever =
+        new MockAspectRetriever(
+            ImmutableMap.of(
+                TEST_PROPERTY_URN,
+                Collections.singletonList(propertySettings),
+                EXISTING_BADGE_URN,
+                Collections.singletonList(propertySettings)));
+    mockGraphRetriever = Mockito.mock(GraphRetriever.class);
+    retrieverContext =
+        io.datahubproject.metadata.context.RetrieverContext.builder()
+            .cachingAspectRetriever(mockAspectRetriever)
+            .searchRetriever(mockSearchRetriever)
+            .graphRetriever(mockGraphRetriever)
+            .build();
+  }
+
+  @Test
+  public void testValidUpsert() {
+
+    // Create settings with showAsAssetBadge = true
+    StructuredPropertySettings propertySettings =
+        new StructuredPropertySettings()
+            .setShowAsAssetBadge(true)
+            .setShowInAssetSummary(true)
+            .setShowInSearchFilters(true);
+
+    Mockito.when(
+            mockSearchRetriever.scroll(
+                Mockito.eq(Collections.singletonList(STRUCTURED_PROPERTY_ENTITY_NAME)),
+                Mockito.any(Filter.class),
+                Mockito.eq(null),
+                Mockito.eq(10)))
+        .thenReturn(new ScrollResult().setEntities(new SearchEntityArray()));
+
+    // Test validation
+    Stream<AspectValidationException> validationResult =
+        ShowPropertyAsBadgeValidator.validateSettingsUpserts(
+            TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY),
+            retrieverContext);
+
+    // Assert no validation exceptions
+    Assert.assertTrue(validationResult.findAny().isEmpty());
+  }
+
+  @Test
+  public void testInvalidUpsertWithExistingBadge() {
+
+    // Create settings with showAsAssetBadge = true
+    StructuredPropertySettings propertySettings =
+        new StructuredPropertySettings()
+            .setShowAsAssetBadge(true)
+            .setShowInAssetSummary(true)
+            .setShowInSearchFilters(true);
+
+    // Mock search results with an existing badge
+    SearchEntity existingBadge = new SearchEntity();
+    existingBadge.setEntity(EXISTING_BADGE_URN);
+    ScrollResult mockResult = new ScrollResult();
+    mockResult.setEntities(new SearchEntityArray(Collections.singletonList(existingBadge)));
+    Mockito.when(
+            mockSearchRetriever.scroll(
+                Mockito.eq(Collections.singletonList(STRUCTURED_PROPERTY_ENTITY_NAME)),
+                Mockito.any(Filter.class),
+                Mockito.eq(null),
+                Mockito.eq(10)))
+        .thenReturn(mockResult);
+
+    // Test validation
+    Stream<AspectValidationException> validationResult =
+        ShowPropertyAsBadgeValidator.validateSettingsUpserts(
+            TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY),
+            retrieverContext);
+
+    // Assert validation exception exists
+    Assert.assertFalse(validationResult.findAny().isEmpty());
+  }
+
+  @Test
+  public void testValidUpsertWithShowAsAssetBadgeFalse() {
+
+    // Create settings with showAsAssetBadge = false
+    StructuredPropertySettings propertySettings =
+        new StructuredPropertySettings()
+            .setShowAsAssetBadge(false)
+            .setShowInAssetSummary(true)
+            .setShowInSearchFilters(true);
+
+    // Mock search results with an existing badge (shouldn't matter since we're setting false)
+    SearchEntity existingBadge = new SearchEntity();
+    existingBadge.setEntity(EXISTING_BADGE_URN);
+    ScrollResult mockResult = new ScrollResult();
+    mockResult.setEntities(new SearchEntityArray(Collections.singletonList(existingBadge)));
+    Mockito.when(
+            mockSearchRetriever.scroll(
+                Mockito.eq(Collections.singletonList(STRUCTURED_PROPERTY_ENTITY_NAME)),
+                Mockito.any(Filter.class),
+                Mockito.eq(null),
+                Mockito.eq(10)))
+        .thenReturn(mockResult);
+
+    // Test validation
+    Stream<AspectValidationException> validationResult =
+        ShowPropertyAsBadgeValidator.validateSettingsUpserts(
+            TestMCP.ofOneUpsertItem(TEST_PROPERTY_URN, propertySettings, TEST_REGISTRY),
+            retrieverContext);
+
+    // Assert no validation exceptions
+    Assert.assertTrue(validationResult.findAny().isEmpty());
+  }
+}
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java
index e8154720a140d..2073f3f01ca90 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java
@@ -99,7 +99,7 @@ public void testGetTimeline() throws Exception {
 
     Map<String, RecordTemplate> latestAspects =
         _entityServiceImpl.getLatestAspectsForUrn(
-            opContext, entityUrn, new HashSet<>(Arrays.asList(aspectName)));
+            opContext, entityUrn, new HashSet<>(Arrays.asList(aspectName)), false);
 
     Set<ChangeCategory> elements = new HashSet<>();
     elements.add(ChangeCategory.TECHNICAL_SCHEMA);
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java
index 88dd81d953947..afa0730483108 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java
@@ -153,6 +153,37 @@ public void testSchemaFieldRename() throws Exception {
         Set.of(SchemaFieldModificationCategory.RENAME.toString()), actual);
   }
 
+  @Test
+  public void testSchemaFieldRename2() throws Exception {
+    SchemaMetadataChangeEventGenerator test = new SchemaMetadataChangeEventGenerator();
+
+    Urn urn = getTestUrn();
+    String entity = "dataset";
+    String aspect = "schemaMetadata";
+    AuditStamp auditStamp = getTestAuditStamp();
+
+    Aspect<SchemaMetadata> from =
+        getSchemaMetadata(
+            List.of(
+                new SchemaField().setFieldPath("id").setNativeDataType("VARCHAR"),
+                new SchemaField().setFieldPath("fullname").setNativeDataType("VARCHAR"),
+                new SchemaField().setFieldPath("LastName").setNativeDataType("VARCHAR")));
+    Aspect<SchemaMetadata> to =
+        getSchemaMetadata(
+            List.of(
+                new SchemaField().setFieldPath("id").setNativeDataType("VARCHAR"),
+                new SchemaField().setFieldPath("fullname").setNativeDataType("VARCHAR"),
+                new SchemaField().setFieldPath("lastName").setNativeDataType("VARCHAR")));
+    List<ChangeEvent> actual = test.getChangeEvents(urn, entity, aspect, from, to, auditStamp);
+    compareDescriptions(
+        Set.of(
+            "A forwards & backwards compatible change due to renaming of the field 'LastName to lastName'."),
+        actual);
+    assertEquals(1, actual.size());
+    compareModificationCategories(
+        Set.of(SchemaFieldModificationCategory.RENAME.toString()), actual);
+  }
+
   @Test
   public void testSchemaFieldDropAdd() throws Exception {
     // When a rename cannot be detected, treated as drop -> add
@@ -204,10 +235,10 @@ public void testSchemaFieldPrimaryKeyChange() throws Exception {
     List<ChangeEvent> actual = test.getChangeEvents(urn, entity, aspect, from, to3, auditStamp);
     compareDescriptions(
         Set.of(
-            "A backwards incompatible change due to a primary key constraint change. "
-                + "The following fields were removed: 'ID'. The following fields were added: 'ID2'."),
+            "A backwards incompatible change due to addition of the primary key field 'ID2'",
+            "A backwards incompatible change due to removal of the primary key field 'ID'"),
         actual);
-    assertEquals(1, actual.size());
+    assertEquals(actual.size(), 2);
     compareModificationCategories(Set.of(SchemaFieldModificationCategory.OTHER.toString()), actual);
   }
 
@@ -274,10 +305,10 @@ public void testSchemaFieldPrimaryKeyChangeRenameAdd() throws Exception {
     List<ChangeEvent> actual = test.getChangeEvents(urn, entity, aspect, from, to3, auditStamp);
     compareDescriptions(
         Set.of(
-            "A backwards incompatible change due to a primary key constraint change. "
-                + "The following fields were removed: 'ID'. The following fields were added: 'ID2'."),
+            "A backwards incompatible change due to addition of the primary key field 'ID2'",
+            "A backwards incompatible change due to removal of the primary key field 'ID'"),
         actual);
-    assertEquals(1, actual.size());
+    assertEquals(actual.size(), 2);
     compareModificationCategories(Set.of(SchemaFieldModificationCategory.OTHER.toString()), actual);
 
     Aspect<SchemaMetadata> to4 =
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
index faf616b0fb3cf..e8420e92e5a5f 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
@@ -26,6 +26,7 @@
 import com.linkedin.data.template.StringMap;
 import com.linkedin.data.template.StringMapArray;
 import com.linkedin.metadata.aspect.EnvelopedAspect;
+import com.linkedin.metadata.config.TimeseriesAspectServiceConfig;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.DataSchemaFactory;
 import com.linkedin.metadata.models.EntitySpec;
@@ -151,7 +152,8 @@ private ElasticSearchTimeseriesAspectService buildService() {
             opContext.getSearchContext().getIndexConvention()),
         getBulkProcessor(),
         1,
-        QueryFilterRewriteChain.EMPTY);
+        QueryFilterRewriteChain.EMPTY,
+        TimeseriesAspectServiceConfig.builder().build());
   }
 
   /*
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceUnitTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceUnitTest.java
index db9d8b450ef7a..9a21c5337db86 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceUnitTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceUnitTest.java
@@ -5,6 +5,7 @@
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.NumericNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.linkedin.metadata.config.TimeseriesAspectServiceConfig;
 import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriteChain;
 import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
@@ -44,7 +45,8 @@ public class TimeseriesAspectServiceUnitTest {
           _timeseriesAspectIndexBuilders,
           _bulkProcessor,
           0,
-          QueryFilterRewriteChain.EMPTY);
+          QueryFilterRewriteChain.EMPTY,
+          TimeseriesAspectServiceConfig.builder().build());
   private final OperationContext opContext =
       TestOperationContexts.systemContextNoSearchAuthorization(_indexConvention);
 
diff --git a/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java b/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java
index 3acd2bf341357..02cd28eb202e9 100644
--- a/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java
+++ b/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java
@@ -171,10 +171,7 @@ public Stream<List<MetadataChangeProposal>> generateMCPs(
                         DefaultAspectsUtil.getAdditionalChanges(
                                 opContext,
                                 AspectsBatchImpl.builder()
-                                    .mcps(
-                                        List.of(mcp),
-                                        auditStamp,
-                                        opContext.getRetrieverContext().get())
+                                    .mcps(List.of(mcp), auditStamp, opContext.getRetrieverContext())
                                     .build()
                                     .getMCPItems(),
                                 entityService,
diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java
index e47cdf80281c9..5e387d7d88292 100644
--- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java
+++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java
@@ -2,11 +2,13 @@
 
 import static com.linkedin.metadata.Constants.*;
 import static io.datahubproject.test.search.config.SearchTestContainerConfiguration.REFRESH_INTERVAL_SECONDS;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anySet;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.metadata.client.JavaEntityClient;
 import com.linkedin.metadata.config.PreProcessHooks;
 import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration;
@@ -302,7 +304,7 @@ private EntityClient entityClientHelper(
             new ConcurrentMapCacheManager(), entitySearchService, 1, false);
 
     AspectDao mockAspectDao = mock(AspectDao.class);
-    when(mockAspectDao.batchGet(anySet()))
+    when(mockAspectDao.batchGet(anySet(), anyBoolean()))
         .thenAnswer(
             args -> {
               Set<EntityAspectIdentifier> ids = args.getArgument(0);
@@ -330,6 +332,6 @@ private EntityClient entityClientHelper(
         null,
         null,
         null,
-        1);
+        EntityClientConfig.builder().batchGetV2Size(1).build());
   }
 }
diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java
index 889473d32d1a3..b7b698c73ddac 100644
--- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java
+++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java
@@ -4,6 +4,7 @@
 import static io.datahubproject.test.search.SearchTestUtils.getGraphQueryConfiguration;
 
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.metadata.client.JavaEntityClient;
 import com.linkedin.metadata.config.DataHubAppConfiguration;
 import com.linkedin.metadata.config.MetadataChangeProposalConfig;
@@ -276,6 +277,6 @@ protected EntityClient entityClient(
         null,
         null,
         null,
-        1);
+        EntityClientConfig.builder().batchGetV2Size(1).build());
   }
 }
diff --git a/metadata-io/src/test/resources/elasticsearch/sample_data/containerindex_v2.json.gz b/metadata-io/src/test/resources/elasticsearch/sample_data/containerindex_v2.json.gz
index 2fa49c810abfa..bd36747255f86 100644
Binary files a/metadata-io/src/test/resources/elasticsearch/sample_data/containerindex_v2.json.gz and b/metadata-io/src/test/resources/elasticsearch/sample_data/containerindex_v2.json.gz differ
diff --git a/metadata-io/src/test/resources/search_config_fixture_test.yml b/metadata-io/src/test/resources/search_config_fixture_test.yml
index 08e713c6b1cd3..e3c97c267188f 100644
--- a/metadata-io/src/test/resources/search_config_fixture_test.yml
+++ b/metadata-io/src/test/resources/search_config_fixture_test.yml
@@ -57,9 +57,9 @@ queryConfigurations:
       boost_mode: replace
 
   # Criteria for exact-match only
-  # Contains quotes, is a single term with `_`, `.`, or `-` (normally consider for tokenization) then use exact match query
+  # Contains quotes
   - queryRegex: >-
-      ^["'].+["']$|^[a-zA-Z0-9]\S+[_.-]\S+[a-zA-Z0-9]$
+      ^["'].+["']$
     simpleQuery: false
     prefixMatchQuery: true
     exactMatchQuery: true
diff --git a/metadata-jobs/common/build.gradle b/metadata-jobs/common/build.gradle
index b0a3a6827b729..120806431b1df 100644
--- a/metadata-jobs/common/build.gradle
+++ b/metadata-jobs/common/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'java-library'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
     implementation(project(':metadata-service:factories')) {
         exclude group: 'org.neo4j.test'
diff --git a/metadata-jobs/mae-consumer-job/build.gradle b/metadata-jobs/mae-consumer-job/build.gradle
index 4980f81d61bb4..1fb46582c0381 100644
--- a/metadata-jobs/mae-consumer-job/build.gradle
+++ b/metadata-jobs/mae-consumer-job/build.gradle
@@ -4,6 +4,7 @@ plugins {
     id 'com.palantir.docker'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
 apply from: "../../gradle/versioning/versioning.gradle"
 
 ext {
diff --git a/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml b/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml
index a9a30916c38c5..474c210c4430b 100644
--- a/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml
+++ b/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml
@@ -3,7 +3,7 @@
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
+            <pattern>%date{ISO8601} [%thread] %-5level %logger{36} urn=%X{entityUrn:-none} aspect=%X{aspectName:-none} entityType=%X{entityType:-none} changeType=%X{changeType:-none} - %msg%n</pattern>
         </encoder>
         <filter class="com.linkedin.metadata.utils.log.LogMessageFilter">
             <excluded>scanned from multiple locations</excluded>
diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle
index 6afc12c30e5f1..b4990e289b10d 100644
--- a/metadata-jobs/mae-consumer/build.gradle
+++ b/metadata-jobs/mae-consumer/build.gradle
@@ -1,7 +1,7 @@
 plugins {
     id 'java-library'
 }
-
+apply from: '../../gradle/coverage/java-coverage.gradle'
 apply plugin: 'pegasus'
 
 configurations {
diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java
index 70b452722abc7..a2d59023ba5ce 100644
--- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java
+++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java
@@ -1,18 +1,27 @@
 package com.linkedin.metadata.kafka;
 
+import static com.linkedin.metadata.Constants.MDC_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.MDC_CHANGE_TYPE;
+import static com.linkedin.metadata.Constants.MDC_ENTITY_TYPE;
+import static com.linkedin.metadata.Constants.MDC_ENTITY_URN;
+
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.Timer;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.EventUtils;
 import com.linkedin.metadata.kafka.hook.MetadataChangeLogHook;
 import com.linkedin.metadata.utils.metrics.MetricUtils;
 import com.linkedin.mxe.MetadataChangeLog;
 import io.datahubproject.metadata.context.OperationContext;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.slf4j.MDC;
 
 @Slf4j
 public class MCLKafkaListener {
@@ -66,17 +75,27 @@ public void consume(final ConsumerRecord<String, GenericRecord> consumerRecord)
         return;
       }
 
+      Urn entityUrn = event.getEntityUrn();
+      String aspectName = event.hasAspectName() ? event.getAspectName() : null;
+      String entityType = event.hasEntityType() ? event.getEntityType() : null;
+      ChangeType changeType = event.hasChangeType() ? event.getChangeType() : null;
+      MDC.put(MDC_ENTITY_URN, Optional.ofNullable(entityUrn).map(Urn::toString).orElse(""));
+      MDC.put(MDC_ASPECT_NAME, aspectName);
+      MDC.put(MDC_ENTITY_TYPE, entityType);
+      MDC.put(
+          MDC_CHANGE_TYPE, Optional.ofNullable(changeType).map(ChangeType::toString).orElse(""));
+
       log.info(
           "Invoking MCL hooks for consumer: {} urn: {}, aspect name: {}, entity type: {}, change type: {}",
           consumerGroupId,
-          event.getEntityUrn(),
-          event.hasAspectName() ? event.getAspectName() : null,
-          event.hasEntityType() ? event.getEntityType() : null,
-          event.hasChangeType() ? event.getChangeType() : null);
+          entityUrn,
+          aspectName,
+          entityType,
+          changeType);
 
       // Here - plug in additional "custom processor hooks"
       for (MetadataChangeLogHook hook : this.hooks) {
-        log.info(
+        log.debug(
             "Invoking MCL hook {} for urn: {}",
             hook.getClass().getSimpleName(),
             event.getEntityUrn());
@@ -98,6 +117,8 @@ public void consume(final ConsumerRecord<String, GenericRecord> consumerRecord)
           "Successfully completed MCL hooks for consumer: {} urn: {}",
           consumerGroupId,
           event.getEntityUrn());
+    } finally {
+      MDC.clear();
     }
   }
 }
diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListenerRegistrar.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListenerRegistrar.java
index fb2880f617d30..c909b0034a912 100644
--- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListenerRegistrar.java
+++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListenerRegistrar.java
@@ -1,5 +1,7 @@
 package com.linkedin.metadata.kafka;
 
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.MCL_EVENT_CONSUMER_NAME;
+
 import com.linkedin.metadata.kafka.config.MetadataChangeLogProcessorCondition;
 import com.linkedin.metadata.kafka.hook.MetadataChangeLogHook;
 import com.linkedin.mxe.Topics;
@@ -39,7 +41,7 @@ public class MCLKafkaListenerRegistrar implements InitializingBean {
   @Autowired private KafkaListenerEndpointRegistry kafkaListenerEndpointRegistry;
 
   @Autowired
-  @Qualifier("kafkaEventConsumer")
+  @Qualifier(MCL_EVENT_CONSUMER_NAME)
   private KafkaListenerContainerFactory<?> kafkaListenerContainerFactory;
 
   @Value("${METADATA_CHANGE_LOG_KAFKA_CONSUMER_GROUP_ID:generic-mae-consumer-job-client}")
diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java
index 771f0880ae75e..c67c786141c25 100644
--- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java
+++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java
@@ -6,6 +6,7 @@
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
 
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.InputField;
@@ -56,6 +57,7 @@
 import com.linkedin.schema.NumberType;
 import com.linkedin.schema.SchemaField;
 import com.linkedin.schema.SchemaFieldDataType;
+import com.linkedin.structured.StructuredPropertyDefinition;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.net.URISyntaxException;
@@ -101,6 +103,7 @@ public class UpdateIndicesHookTest {
   private Urn actorUrn;
   private UpdateIndicesService updateIndicesService;
   private UpdateIndicesHook reprocessUIHook;
+  private OperationContext opContext;
 
   @Value("${elasticsearch.index.maxArrayLength}")
   private int maxArrayLength;
@@ -133,13 +136,12 @@ public void setupTest() {
             mockEntityIndexBuilders,
             "MD5");
 
-    OperationContext systemOperationContext =
-        TestOperationContexts.systemContextNoSearchAuthorization();
+    opContext = TestOperationContexts.systemContextNoSearchAuthorization();
 
     updateIndicesHook = new UpdateIndicesHook(updateIndicesService, true, false);
-    updateIndicesHook.init(systemOperationContext);
+    updateIndicesHook.init(opContext);
     reprocessUIHook = new UpdateIndicesHook(updateIndicesService, true, true);
-    reprocessUIHook.init(systemOperationContext);
+    reprocessUIHook.init(opContext);
   }
 
   @Test
@@ -478,6 +480,38 @@ public void testMCLUIPreProcessedReprocess() throws Exception {
         .upsertDocument(any(OperationContext.class), Mockito.any(), Mockito.any(), Mockito.any());
   }
 
+  @Test
+  public void testUpdateIndexMappings() throws CloneNotSupportedException {
+    // ensure no mutation
+    EntitySpec entitySpec =
+        opContext.getEntityRegistry().getEntitySpec(STRUCTURED_PROPERTY_ENTITY_NAME);
+    AspectSpec aspectSpec = entitySpec.getAspectSpec(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME);
+
+    StructuredPropertyDefinition oldValueOrigin =
+        new StructuredPropertyDefinition()
+            .setEntityTypes(
+                new UrnArray(
+                    UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,foo1,PROD)"),
+                    UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,foo2,PROD)")));
+    StructuredPropertyDefinition newValueOrigin =
+        new StructuredPropertyDefinition()
+            .setEntityTypes(
+                new UrnArray(
+                    UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,foo2,PROD)"),
+                    UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,foo3,PROD)")));
+
+    StructuredPropertyDefinition oldValue =
+        new StructuredPropertyDefinition(oldValueOrigin.data().copy());
+    StructuredPropertyDefinition newValue =
+        new StructuredPropertyDefinition(newValueOrigin.data().copy());
+
+    updateIndicesService.updateIndexMappings(
+        UrnUtils.getUrn(TEST_DATASET_URN), entitySpec, aspectSpec, newValue, oldValue);
+
+    assertEquals(oldValue, oldValueOrigin, "Ensure no mutation to input objects");
+    assertEquals(newValue, newValueOrigin, "Ensure no mutation to input objects");
+  }
+
   private EntityRegistry createMockEntityRegistry() {
     // need to mock this registry instead of using test-entity-registry.yml because inputFields does
     // not work due to a known bug
diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLMAESpringTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLMAESpringTest.java
index d613c7d4af879..8ee3560a0cb74 100644
--- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLMAESpringTest.java
+++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLMAESpringTest.java
@@ -4,6 +4,7 @@
 import static org.testng.AssertJUnit.assertNotNull;
 import static org.testng.AssertJUnit.assertTrue;
 
+import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
 import com.linkedin.gms.factory.config.ConfigurationProvider;
 import com.linkedin.metadata.kafka.MCLKafkaListenerRegistrar;
 import com.linkedin.metadata.kafka.hook.UpdateIndicesHook;
@@ -36,6 +37,12 @@ public class MCLMAESpringTest extends AbstractTestNGSpringContextTests {
 
   @Autowired private UpdateIndicesService updateIndicesService;
 
+  static {
+    PathSpecBasedSchemaAnnotationVisitor.class
+        .getClassLoader()
+        .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false);
+  }
+
   @Test
   public void testHooks() {
     MCLKafkaListenerRegistrar registrar =
diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringCommonTestConfiguration.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringCommonTestConfiguration.java
index b34bb5bd0e0a8..f16c9dbd82e74 100644
--- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringCommonTestConfiguration.java
+++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringCommonTestConfiguration.java
@@ -5,6 +5,7 @@
 
 import com.datahub.authentication.Authentication;
 import com.datahub.metadata.ingestion.IngestionScheduler;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.gms.factory.plugins.SpringStandardPluginConfiguration;
 import com.linkedin.metadata.boot.kafka.DataHubUpgradeKafkaListener;
@@ -19,8 +20,8 @@
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.OperationContextConfig;
-import io.datahubproject.metadata.context.RetrieverContext;
 import io.datahubproject.metadata.context.ServicesRegistryContext;
+import io.datahubproject.metadata.context.ValidationContext;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import org.apache.avro.generic.GenericRecord;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -57,6 +58,11 @@ public class MCLSpringCommonTestConfiguration {
 
   @MockBean public IngestionScheduler ingestionScheduler;
 
+  @Bean
+  public EntityClientConfig entityClientConfig() {
+    return EntityClientConfig.builder().build();
+  }
+
   @MockBean(name = "systemEntityClient")
   public SystemEntityClient systemEntityClient;
 
@@ -88,7 +94,8 @@ public OperationContext operationContext(
         entityRegistry,
         mock(ServicesRegistryContext.class),
         indexConvention,
-        mock(RetrieverContext.class));
+        TestOperationContexts.emptyActiveUsersRetrieverContext(() -> entityRegistry),
+        mock(ValidationContext.class));
   }
 
   @MockBean SpringStandardPluginConfiguration springStandardPluginConfiguration;
diff --git a/metadata-jobs/mce-consumer-job/build.gradle b/metadata-jobs/mce-consumer-job/build.gradle
index 49c69626c211c..d9b8965d7c8ac 100644
--- a/metadata-jobs/mce-consumer-job/build.gradle
+++ b/metadata-jobs/mce-consumer-job/build.gradle
@@ -4,6 +4,7 @@ plugins {
     id 'com.palantir.docker'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
 apply from: "../../gradle/versioning/versioning.gradle"
 
 ext {
diff --git a/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml b/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml
index f94088735fa29..f8674dba82916 100644
--- a/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml
+++ b/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml
@@ -3,7 +3,7 @@
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
+            <pattern>%date{ISO8601} [%thread] %-5level %logger{36} urn=%X{entityUrn:-none} aspect=%X{aspectName:-none} entityType=%X{entityType:-none} changeType=%X{changeType:-none} - %msg%n</pattern>
         </encoder>
         <filter class="com.linkedin.metadata.utils.log.LogMessageFilter">
             <excluded>scanned from multiple locations</excluded>
diff --git a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java
index ba650c25a6117..f65e803a499b0 100644
--- a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java
+++ b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java
@@ -1,5 +1,6 @@
 package com.linkedin.metadata.kafka;
 
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.entity.client.SystemRestliEntityClient;
 import com.linkedin.gms.factory.auth.SystemAuthenticationFactory;
@@ -39,16 +40,25 @@ public class MceConsumerApplicationTestConfiguration {
   @Bean
   @Primary
   public SystemEntityClient systemEntityClient(
-      @Qualifier("configurationProvider") final ConfigurationProvider configurationProvider) {
+      @Qualifier("configurationProvider") final ConfigurationProvider configurationProvider,
+      final EntityClientConfig entityClientConfig) {
     String selfUri = restTemplate.getRootUri();
     final Client restClient = DefaultRestliClientFactory.getRestLiClient(URI.create(selfUri), null);
     return new SystemRestliEntityClient(
         restClient,
-        new ExponentialBackoff(1),
-        1,
-        configurationProvider.getCache().getClient().getEntityClient(),
-        1,
-        2);
+        entityClientConfig,
+        configurationProvider.getCache().getClient().getEntityClient());
+  }
+
+  @Bean
+  @Primary
+  public EntityClientConfig entityClientConfig() {
+    return EntityClientConfig.builder()
+        .backoffPolicy(new ExponentialBackoff(1))
+        .retryCount(1)
+        .batchGetV2Size(1)
+        .batchGetV2Concurrency(2)
+        .build();
   }
 
   @MockBean public Database ebeanServer;
diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle
index 5ea24059a3ee3..2da3957c4bb21 100644
--- a/metadata-jobs/mce-consumer/build.gradle
+++ b/metadata-jobs/mce-consumer/build.gradle
@@ -3,6 +3,7 @@ plugins {
     id 'pegasus'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
 
 configurations {
     avro
diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java
index 1b3d19915b439..5d2f6452e6919 100644
--- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java
+++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java
@@ -1,5 +1,7 @@
 package com.linkedin.metadata.kafka;
 
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.DEFAULT_EVENT_CONSUMER_NAME;
+
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.Timer;
@@ -60,7 +62,7 @@ public class MetadataChangeEventsProcessor {
           "${METADATA_CHANGE_EVENT_NAME:${KAFKA_MCE_TOPIC_NAME:"
               + Topics.METADATA_CHANGE_EVENT
               + "}}",
-      containerFactory = "kafkaEventConsumer")
+      containerFactory = DEFAULT_EVENT_CONSUMER_NAME)
   @Deprecated
   public void consume(final ConsumerRecord<String, GenericRecord> consumerRecord) {
     try (Timer.Context i = MetricUtils.timer(this.getClass(), "consume").time()) {
diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java
index 2f3f35697e476..4e356f5fb3670 100644
--- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java
+++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java
@@ -1,32 +1,36 @@
 package com.linkedin.metadata.kafka;
 
+import static com.linkedin.metadata.Constants.MDC_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.MDC_CHANGE_TYPE;
+import static com.linkedin.metadata.Constants.MDC_ENTITY_TYPE;
+import static com.linkedin.metadata.Constants.MDC_ENTITY_URN;
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.MCP_EVENT_CONSUMER_NAME;
+
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.Timer;
+import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.client.SystemEntityClient;
+import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.gms.factory.config.ConfigurationProvider;
 import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory;
 import com.linkedin.metadata.EventUtils;
-import com.linkedin.metadata.dao.throttle.ThrottleControl;
 import com.linkedin.metadata.dao.throttle.ThrottleSensor;
 import com.linkedin.metadata.kafka.config.MetadataChangeProposalProcessorCondition;
+import com.linkedin.metadata.kafka.util.KafkaListenerUtil;
 import com.linkedin.metadata.utils.metrics.MetricUtils;
-import com.linkedin.mxe.FailedMetadataChangeProposal;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.mxe.Topics;
 import io.datahubproject.metadata.context.OperationContext;
-import java.io.IOException;
 import java.util.Optional;
-import javax.annotation.Nonnull;
 import javax.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.generic.IndexedRecord;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerRecord;
+import org.slf4j.MDC;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Conditional;
@@ -34,7 +38,6 @@
 import org.springframework.kafka.annotation.EnableKafka;
 import org.springframework.kafka.annotation.KafkaListener;
 import org.springframework.kafka.config.KafkaListenerEndpointRegistry;
-import org.springframework.kafka.listener.MessageListenerContainer;
 import org.springframework.stereotype.Component;
 
 @Slf4j
@@ -71,44 +74,13 @@ public class MetadataChangeProposalsProcessor {
 
   @PostConstruct
   public void registerConsumerThrottle() {
-    if (kafkaThrottle != null
-        && provider
-            .getMetadataChangeProposal()
-            .getThrottle()
-            .getComponents()
-            .getMceConsumer()
-            .isEnabled()) {
-      log.info("MCE Consumer Throttle Enabled");
-      kafkaThrottle.addCallback(
-          (throttleEvent) -> {
-            Optional<MessageListenerContainer> container =
-                Optional.ofNullable(registry.getListenerContainer(mceConsumerGroupId));
-            if (container.isEmpty()) {
-              log.warn(
-                  "Expected container was missing: {} throttle is not possible.",
-                  mceConsumerGroupId);
-            } else {
-              if (throttleEvent.isThrottled()) {
-                container.ifPresent(MessageListenerContainer::pause);
-                return ThrottleControl.builder()
-                    // resume consumer after sleep
-                    .callback(
-                        (resumeEvent) -> container.ifPresent(MessageListenerContainer::resume))
-                    .build();
-              }
-            }
-
-            return ThrottleControl.NONE;
-          });
-    } else {
-      log.info("MCE Consumer Throttle Disabled");
-    }
+    KafkaListenerUtil.registerThrottle(kafkaThrottle, provider, registry, mceConsumerGroupId);
   }
 
   @KafkaListener(
       id = CONSUMER_GROUP_ID_VALUE,
       topics = "${METADATA_CHANGE_PROPOSAL_TOPIC_NAME:" + Topics.METADATA_CHANGE_PROPOSAL + "}",
-      containerFactory = "kafkaEventConsumer")
+      containerFactory = MCP_EVENT_CONSUMER_NAME)
   public void consume(final ConsumerRecord<String, GenericRecord> consumerRecord) {
     try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "consume").time()) {
       kafkaLagStats.update(System.currentTimeMillis() - consumerRecord.timestamp());
@@ -123,48 +95,36 @@ public void consume(final ConsumerRecord<String, GenericRecord> consumerRecord)
           consumerRecord.serializedValueSize(),
           consumerRecord.timestamp());
 
-      log.debug("Record {}", record);
+      if (log.isDebugEnabled()) {
+        log.debug("Record {}", record);
+      }
 
       MetadataChangeProposal event = new MetadataChangeProposal();
       try {
         event = EventUtils.avroToPegasusMCP(record);
-        log.debug("MetadataChangeProposal {}", event);
-        // TODO: Get this from the event itself.
+
+        Urn entityUrn = event.getEntityUrn();
+        String aspectName = event.hasAspectName() ? event.getAspectName() : null;
+        String entityType = event.hasEntityType() ? event.getEntityType() : null;
+        ChangeType changeType = event.hasChangeType() ? event.getChangeType() : null;
+        MDC.put(MDC_ENTITY_URN, Optional.ofNullable(entityUrn).map(Urn::toString).orElse(""));
+        MDC.put(MDC_ASPECT_NAME, aspectName);
+        MDC.put(MDC_ENTITY_TYPE, entityType);
+        MDC.put(
+            MDC_CHANGE_TYPE, Optional.ofNullable(changeType).map(ChangeType::toString).orElse(""));
+
+        if (log.isDebugEnabled()) {
+          log.debug("MetadataChangeProposal {}", event);
+        }
         String urn = entityClient.ingestProposal(systemOperationContext, event, false);
         log.info("Successfully processed MCP event urn: {}", urn);
       } catch (Throwable throwable) {
         log.error("MCP Processor Error", throwable);
         log.error("Message: {}", record);
-        sendFailedMCP(event, throwable);
+        KafkaListenerUtil.sendFailedMCP(event, throwable, fmcpTopicName, kafkaProducer);
       }
+    } finally {
+      MDC.clear();
     }
   }
-
-  private void sendFailedMCP(@Nonnull MetadataChangeProposal event, @Nonnull Throwable throwable) {
-    final FailedMetadataChangeProposal failedMetadataChangeProposal =
-        createFailedMCPEvent(event, throwable);
-    try {
-      final GenericRecord genericFailedMCERecord =
-          EventUtils.pegasusToAvroFailedMCP(failedMetadataChangeProposal);
-      log.debug("Sending FailedMessages to topic - {}", fmcpTopicName);
-      log.info(
-          "Error while processing FMCP: FailedMetadataChangeProposal - {}",
-          failedMetadataChangeProposal);
-      kafkaProducer.send(new ProducerRecord<>(fmcpTopicName, genericFailedMCERecord));
-    } catch (IOException e) {
-      log.error(
-          "Error while sending FailedMetadataChangeProposal: Exception  - {}, FailedMetadataChangeProposal - {}",
-          e.getStackTrace(),
-          failedMetadataChangeProposal);
-    }
-  }
-
-  @Nonnull
-  private FailedMetadataChangeProposal createFailedMCPEvent(
-      @Nonnull MetadataChangeProposal event, @Nonnull Throwable throwable) {
-    final FailedMetadataChangeProposal fmcp = new FailedMetadataChangeProposal();
-    fmcp.setError(ExceptionUtils.getStackTrace(throwable));
-    fmcp.setMetadataChangeProposal(event);
-    return fmcp;
-  }
 }
diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/batch/BatchMetadataChangeProposalsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/batch/BatchMetadataChangeProposalsProcessor.java
new file mode 100644
index 0000000000000..fed93628fe4d7
--- /dev/null
+++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/batch/BatchMetadataChangeProposalsProcessor.java
@@ -0,0 +1,116 @@
+package com.linkedin.metadata.kafka.batch;
+
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.linkedin.entity.client.SystemEntityClient;
+import com.linkedin.gms.factory.config.ConfigurationProvider;
+import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory;
+import com.linkedin.metadata.EventUtils;
+import com.linkedin.metadata.dao.throttle.ThrottleSensor;
+import com.linkedin.metadata.kafka.config.batch.BatchMetadataChangeProposalProcessorCondition;
+import com.linkedin.metadata.kafka.util.KafkaListenerUtil;
+import com.linkedin.metadata.utils.metrics.MetricUtils;
+import com.linkedin.mxe.MetadataChangeProposal;
+import com.linkedin.mxe.Topics;
+import io.datahubproject.metadata.context.OperationContext;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.producer.Producer;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.context.annotation.Import;
+import org.springframework.kafka.annotation.EnableKafka;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.config.KafkaListenerEndpointRegistry;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+@Import({RestliEntityClientFactory.class})
+@Conditional(BatchMetadataChangeProposalProcessorCondition.class)
+@EnableKafka
+@RequiredArgsConstructor
+public class BatchMetadataChangeProposalsProcessor {
+  private static final String CONSUMER_GROUP_ID_VALUE =
+      "${METADATA_CHANGE_PROPOSAL_KAFKA_CONSUMER_GROUP_ID:generic-mce-consumer-job-client}";
+
+  private final OperationContext systemOperationContext;
+  private final SystemEntityClient entityClient;
+  private final Producer<String, IndexedRecord> kafkaProducer;
+
+  @Qualifier("kafkaThrottle")
+  private final ThrottleSensor kafkaThrottle;
+
+  private final KafkaListenerEndpointRegistry registry;
+  private final ConfigurationProvider provider;
+
+  private final Histogram kafkaLagStats =
+      MetricUtils.get().histogram(MetricRegistry.name(this.getClass(), "kafkaLag"));
+
+  @Value(
+      "${FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME:"
+          + Topics.FAILED_METADATA_CHANGE_PROPOSAL
+          + "}")
+  private String fmcpTopicName;
+
+  @Value(CONSUMER_GROUP_ID_VALUE)
+  private String mceConsumerGroupId;
+
+  @PostConstruct
+  public void registerConsumerThrottle() {
+    KafkaListenerUtil.registerThrottle(kafkaThrottle, provider, registry, mceConsumerGroupId);
+  }
+
+  @KafkaListener(
+      id = CONSUMER_GROUP_ID_VALUE,
+      topics = "${METADATA_CHANGE_PROPOSAL_TOPIC_NAME:" + Topics.METADATA_CHANGE_PROPOSAL + "}",
+      containerFactory = "kafkaEventConsumer",
+      batch = "true")
+  public void consume(final List<ConsumerRecord<String, GenericRecord>> consumerRecords) {
+    try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "consume").time()) {
+      List<MetadataChangeProposal> metadataChangeProposals =
+          new ArrayList<>(consumerRecords.size());
+      for (ConsumerRecord<String, GenericRecord> consumerRecord : consumerRecords) {
+        kafkaLagStats.update(System.currentTimeMillis() - consumerRecord.timestamp());
+        final GenericRecord record = consumerRecord.value();
+
+        log.info(
+            "Got MCP event key: {}, topic: {}, partition: {}, offset: {}, value size: {}, timestamp: {}",
+            consumerRecord.key(),
+            consumerRecord.topic(),
+            consumerRecord.partition(),
+            consumerRecord.offset(),
+            consumerRecord.serializedValueSize(),
+            consumerRecord.timestamp());
+
+        MetadataChangeProposal event = new MetadataChangeProposal();
+        try {
+          event = EventUtils.avroToPegasusMCP(record);
+        } catch (Throwable throwable) {
+          log.error("MCP Processor Error", throwable);
+          log.error("Message: {}", record);
+          KafkaListenerUtil.sendFailedMCP(event, throwable, fmcpTopicName, kafkaProducer);
+        }
+        metadataChangeProposals.add(event);
+      }
+
+      try {
+        List<String> urns =
+            entityClient.batchIngestProposals(
+                systemOperationContext, metadataChangeProposals, false);
+        log.info("Successfully processed MCP event urns: {}", urns);
+      } catch (Exception e) {
+        // Java client should never throw this
+        log.error("Exception in batch ingest", e);
+      }
+    }
+  }
+}
diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeProposalProcessorCondition.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeProposalProcessorCondition.java
index 1cdb05b04e0ac..554684d5e8fe7 100644
--- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeProposalProcessorCondition.java
+++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeProposalProcessorCondition.java
@@ -9,7 +9,8 @@ public class MetadataChangeProposalProcessorCondition implements Condition {
   @Override
   public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
     Environment env = context.getEnvironment();
-    return "true".equals(env.getProperty("MCE_CONSUMER_ENABLED"))
-        || "true".equals(env.getProperty("MCP_CONSUMER_ENABLED"));
+    return ("true".equals(env.getProperty("MCE_CONSUMER_ENABLED"))
+            || "true".equals(env.getProperty("MCP_CONSUMER_ENABLED")))
+        && !Boolean.parseBoolean(env.getProperty("MCP_CONSUMER_BATCH_ENABLED"));
   }
 }
diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/batch/BatchMetadataChangeProposalProcessorCondition.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/batch/BatchMetadataChangeProposalProcessorCondition.java
new file mode 100644
index 0000000000000..296e37c7a9069
--- /dev/null
+++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/batch/BatchMetadataChangeProposalProcessorCondition.java
@@ -0,0 +1,16 @@
+package com.linkedin.metadata.kafka.config.batch;
+
+import org.springframework.context.annotation.Condition;
+import org.springframework.context.annotation.ConditionContext;
+import org.springframework.core.env.Environment;
+import org.springframework.core.type.AnnotatedTypeMetadata;
+
+public class BatchMetadataChangeProposalProcessorCondition implements Condition {
+  @Override
+  public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
+    Environment env = context.getEnvironment();
+    return ("true".equals(env.getProperty("MCE_CONSUMER_ENABLED"))
+            || "true".equals(env.getProperty("MCP_CONSUMER_ENABLED")))
+        && Boolean.parseBoolean(env.getProperty("MCP_CONSUMER_BATCH_ENABLED"));
+  }
+}
diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/util/KafkaListenerUtil.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/util/KafkaListenerUtil.java
new file mode 100644
index 0000000000000..874a45c995e91
--- /dev/null
+++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/util/KafkaListenerUtil.java
@@ -0,0 +1,96 @@
+package com.linkedin.metadata.kafka.util;
+
+import com.linkedin.gms.factory.config.ConfigurationProvider;
+import com.linkedin.metadata.EventUtils;
+import com.linkedin.metadata.dao.throttle.ThrottleControl;
+import com.linkedin.metadata.dao.throttle.ThrottleSensor;
+import com.linkedin.mxe.FailedMetadataChangeProposal;
+import com.linkedin.mxe.MetadataChangeProposal;
+import java.io.IOException;
+import java.util.Optional;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.springframework.kafka.config.KafkaListenerEndpointRegistry;
+import org.springframework.kafka.listener.MessageListenerContainer;
+
+@Slf4j
+public class KafkaListenerUtil {
+
+  private KafkaListenerUtil() {}
+
+  public static void registerThrottle(
+      ThrottleSensor kafkaThrottle,
+      ConfigurationProvider provider,
+      KafkaListenerEndpointRegistry registry,
+      String mceConsumerGroupId) {
+    if (kafkaThrottle != null
+        && provider
+            .getMetadataChangeProposal()
+            .getThrottle()
+            .getComponents()
+            .getMceConsumer()
+            .isEnabled()) {
+      log.info("MCE Consumer Throttle Enabled");
+      kafkaThrottle.addCallback(
+          (throttleEvent) -> {
+            Optional<MessageListenerContainer> container =
+                Optional.ofNullable(registry.getListenerContainer(mceConsumerGroupId));
+            if (container.isEmpty()) {
+              log.warn(
+                  "Expected container was missing: {} throttle is not possible.",
+                  mceConsumerGroupId);
+            } else {
+              if (throttleEvent.isThrottled()) {
+                container.ifPresent(MessageListenerContainer::pause);
+                return ThrottleControl.builder()
+                    // resume consumer after sleep
+                    .callback(
+                        (resumeEvent) -> container.ifPresent(MessageListenerContainer::resume))
+                    .build();
+              }
+            }
+
+            return ThrottleControl.NONE;
+          });
+    } else {
+      log.info("MCE Consumer Throttle Disabled");
+    }
+  }
+
+  public static void sendFailedMCP(
+      @Nonnull MetadataChangeProposal event,
+      @Nonnull Throwable throwable,
+      String fmcpTopicName,
+      Producer<String, IndexedRecord> kafkaProducer) {
+    final FailedMetadataChangeProposal failedMetadataChangeProposal =
+        createFailedMCPEvent(event, throwable);
+    try {
+      final GenericRecord genericFailedMCERecord =
+          EventUtils.pegasusToAvroFailedMCP(failedMetadataChangeProposal);
+      log.debug("Sending FailedMessages to topic - {}", fmcpTopicName);
+      log.info(
+          "Error while processing FMCP: FailedMetadataChangeProposal - {}",
+          failedMetadataChangeProposal);
+      kafkaProducer.send(new ProducerRecord<>(fmcpTopicName, genericFailedMCERecord));
+    } catch (IOException e) {
+      log.error(
+          "Error while sending FailedMetadataChangeProposal: Exception  - {}, FailedMetadataChangeProposal - {}",
+          e.getStackTrace(),
+          failedMetadataChangeProposal);
+    }
+  }
+
+  @Nonnull
+  public static FailedMetadataChangeProposal createFailedMCPEvent(
+      @Nonnull MetadataChangeProposal event, @Nonnull Throwable throwable) {
+    final FailedMetadataChangeProposal fmcp = new FailedMetadataChangeProposal();
+    fmcp.setError(ExceptionUtils.getStackTrace(throwable));
+    fmcp.setMetadataChangeProposal(event);
+    return fmcp;
+  }
+}
diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle
index 35fdf0231c406..ad17a44377085 100644
--- a/metadata-jobs/pe-consumer/build.gradle
+++ b/metadata-jobs/pe-consumer/build.gradle
@@ -3,6 +3,8 @@ plugins {
   id 'pegasus'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 configurations {
   avro
 }
diff --git a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java
index 358a2ac0c2ee3..5d11697bed93d 100644
--- a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java
+++ b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java
@@ -1,5 +1,7 @@
 package com.datahub.event;
 
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.PE_EVENT_CONSUMER_NAME;
+
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.Timer;
@@ -56,7 +58,7 @@ public PlatformEventProcessor(
   @KafkaListener(
       id = "${PLATFORM_EVENT_KAFKA_CONSUMER_GROUP_ID:generic-platform-event-job-client}",
       topics = {"${PLATFORM_EVENT_TOPIC_NAME:" + Topics.PLATFORM_EVENT + "}"},
-      containerFactory = "kafkaEventConsumer")
+      containerFactory = PE_EVENT_CONSUMER_NAME)
   public void consume(final ConsumerRecord<String, GenericRecord> consumerRecord) {
     try (Timer.Context i = MetricUtils.timer(this.getClass(), "consume").time()) {
 
diff --git a/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java
index 47740b02d6166..65ee6b8591f48 100644
--- a/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java
+++ b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java
@@ -93,8 +93,6 @@ public void testMCLOnBusinessAttributeUpdate() throws Exception {
                 new RelatedEntity(BUSINESS_ATTRIBUTE_OF, SCHEMA_FIELD_URN.toString())));
 
     when(opContext
-            .getRetrieverContext()
-            .get()
             .getAspectRetriever()
             .getLatestAspectObjects(
                 eq(Set.of(SCHEMA_FIELD_URN)), eq(Set.of(BUSINESS_ATTRIBUTE_ASPECT))))
@@ -108,7 +106,7 @@ public void testMCLOnBusinessAttributeUpdate() throws Exception {
 
     // verify
     // page 1
-    Mockito.verify(opContext.getRetrieverContext().get().getGraphRetriever(), Mockito.times(1))
+    Mockito.verify(opContext.getRetrieverContext().getGraphRetriever(), Mockito.times(1))
         .scrollRelatedEntities(
             isNull(),
             any(Filter.class),
@@ -122,7 +120,7 @@ public void testMCLOnBusinessAttributeUpdate() throws Exception {
             isNull(),
             isNull());
     // page 2
-    Mockito.verify(opContext.getRetrieverContext().get().getGraphRetriever(), Mockito.times(1))
+    Mockito.verify(opContext.getRetrieverContext().getGraphRetriever(), Mockito.times(1))
         .scrollRelatedEntities(
             isNull(),
             any(Filter.class),
@@ -136,7 +134,7 @@ public void testMCLOnBusinessAttributeUpdate() throws Exception {
             isNull(),
             isNull());
 
-    Mockito.verifyNoMoreInteractions(opContext.getRetrieverContext().get().getGraphRetriever());
+    Mockito.verifyNoMoreInteractions(opContext.getRetrieverContext().getGraphRetriever());
 
     // 2 pages = 2 ingest proposals
     Mockito.verify(mockUpdateIndicesService, Mockito.times(2))
@@ -152,8 +150,8 @@ private void testMCLOnInvalidCategory() throws Exception {
     businessAttributeServiceHook.handleChangeEvent(opContext, platformEvent);
 
     // verify
-    Mockito.verifyNoInteractions(opContext.getRetrieverContext().get().getGraphRetriever());
-    Mockito.verifyNoInteractions(opContext.getAspectRetrieverOpt().get());
+    Mockito.verifyNoInteractions(opContext.getRetrieverContext().getGraphRetriever());
+    Mockito.verifyNoInteractions(opContext.getAspectRetriever());
     Mockito.verifyNoInteractions(mockUpdateIndicesService);
   }
 
@@ -226,13 +224,15 @@ private OperationContext mockOperationContextWithGraph(List<RelatedEntity> graph
 
     RetrieverContext mockRetrieverContext = mock(RetrieverContext.class);
     when(mockRetrieverContext.getAspectRetriever()).thenReturn(mock(AspectRetriever.class));
+    when(mockRetrieverContext.getCachingAspectRetriever())
+        .thenReturn(TestOperationContexts.emptyActiveUsersAspectRetriever(null));
     when(mockRetrieverContext.getGraphRetriever()).thenReturn(graphRetriever);
 
     OperationContext opContext =
         TestOperationContexts.systemContextNoSearchAuthorization(mockRetrieverContext);
 
     // reset mock for test
-    reset(opContext.getAspectRetrieverOpt().get());
+    reset(opContext.getAspectRetriever());
 
     if (!graphEdges.isEmpty()) {
 
diff --git a/metadata-models-custom/README.md b/metadata-models-custom/README.md
index 1d26251bc13c9..83917530a41d0 100644
--- a/metadata-models-custom/README.md
+++ b/metadata-models-custom/README.md
@@ -204,6 +204,19 @@ that were loaded for debugging purposes.
 }
 ```
 
+#### Custom Plugin Ecosystem Overview
+
+The following diagram shows the overall picture of the various validators, mutators, and side effects shown within
+the context of typical read/write operations within DataHub. Each component is discussed in further detail in the 
+sections below.
+
+<p align="center">
+  <img width="80%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/metadata-models-custom/custom-plugin-overview.svg"/>
+</p>
+
+In the diagram above, the circles represent Aspects (custom aspects or standard). As the Aspects progress they can be mutated/changed,
+rejected, or additional aspects can be generated by side effects.
+
 #### Custom Validators
 
 Custom aspects might require that instances of those aspects adhere to specific conditions or rules. These conditions could vary wildly depending on the use case however they could be as simple
diff --git a/metadata-models-custom/build.gradle b/metadata-models-custom/build.gradle
index 412c19194c733..9babf9467e5d6 100644
--- a/metadata-models-custom/build.gradle
+++ b/metadata-models-custom/build.gradle
@@ -19,6 +19,8 @@ plugins {
     id 'pegasus'
 }
 
+apply from: '../gradle/coverage/java-coverage.gradle'
+
 if (project.hasProperty('projVersion')) {
   project.version = project.projVersion
 } else {
diff --git a/metadata-models-validator/build.gradle b/metadata-models-validator/build.gradle
index 1dae53e817ae1..c828ab041c27b 100644
--- a/metadata-models-validator/build.gradle
+++ b/metadata-models-validator/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java'
 }
 
+apply from: '../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(":entity-registry")
   implementation spec.product.pegasus.data
diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle
index 179e1eac177ac..2d0b433d69013 100644
--- a/metadata-models/build.gradle
+++ b/metadata-models/build.gradle
@@ -6,14 +6,18 @@ plugins {
   id 'org.hidetake.swagger.generator'
 }
 
+apply from: '../gradle/coverage/java-coverage.gradle'
 
 dependencies {
-  api spec.product.pegasus.data
-    constraints {
-      implementation('org.apache.commons:commons-text:1.10.0') {
-        because 'Vulnerability Issue'
-      }
+  constraints {
+    implementation('org.apache.commons:commons-text:1.10.0') {
+      because 'Vulnerability Issue'
     }
+  }
+
+  api(spec.product.pegasus.data) {
+    exclude group: 'javax.servlet', module: 'javax.servlet-api'
+  }
   api project(':li-utils')
   api project(path: ':li-utils', configuration: "dataTemplate")
   dataModel project(':li-utils')
@@ -25,7 +29,7 @@ dependencies {
 
   compileOnly externalDependency.lombok
   annotationProcessor externalDependency.lombok
-  api externalDependency.swaggerAnnotations
+  compileOnly externalDependency.swaggerAnnotations
   compileOnly externalDependency.jacksonCore
   compileOnly externalDependency.jacksonDataBind
 
diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl
index f33c41e63efed..fe782dbe01ca9 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl
@@ -15,7 +15,7 @@ record DataProcessInstanceOutput {
   @Relationship = {
     "/*": {
       "name": "Produces",
-      "entityTypes": [ "dataset" ]
+      "entityTypes": [ "dataset", "mlModel" ]
     }
   }
   @Searchable = {
diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceProperties.pdl
index c63cb1a97c017..5c6bfaecf1ef4 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceProperties.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceProperties.pdl
@@ -52,4 +52,4 @@ record DataProcessInstanceProperties includes CustomProperties, ExternalReferenc
   }
   created: AuditStamp
 
-}
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl
index d857c7ff611e3..6a87c3d63cbd9 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl
@@ -7,5 +7,6 @@ record EntityTypeKey {
     /**
      * A unique id for an entity type. Usually this will be a unique namespace + entity name.
      */
+    @Searchable = {}
     id: string
 }
diff --git a/metadata-models/src/main/pegasus/com/linkedin/form/FormPrompt.pdl b/metadata-models/src/main/pegasus/com/linkedin/form/FormPrompt.pdl
index f84a8a719f07a..55e1a2a8fc1b7 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/form/FormPrompt.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/form/FormPrompt.pdl
@@ -43,6 +43,10 @@ record FormPrompt {
     /**
      * The structured property that is required on this entity
      */
+    @Searchable = {
+      "fieldType": "URN",
+      "fieldName": "structuredPropertyPromptUrns",
+    }
     urn: Urn
   }
 
diff --git a/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl
index 37e85b6e542bd..3d384bbd6b08f 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl
@@ -21,6 +21,9 @@ record DataHubIngestionSourceInfo {
   /**
    * The type of the source itself, e.g. mysql, bigquery, bigquery-usage. Should match the recipe.
    */
+  @Searchable = {
+   "fieldType": "TEXT_PARTIAL"
+  }
   type: string
 
   /**
@@ -50,6 +53,9 @@ record DataHubIngestionSourceInfo {
      /**
       * The id of the executor to use to execute the ingestion run
       */
+      @Searchable = {
+         "fieldName": "sourceExecutorId"
+      }
       executorId: optional string
 
      /**
diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/SearchFlags.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/SearchFlags.pdl
index a3d2067ae5db2..a3a7a8cda58a8 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/SearchFlags.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/SearchFlags.pdl
@@ -58,4 +58,10 @@ record SearchFlags {
    * invoke query rewrite chain for filters based on configured rewriters
    */
    rewriteQuery: optional boolean = true
+
+   /**
+    * Include default facets when getting facets to aggregate on in search requests.
+    * By default we include these, but custom aggregation requests don't need them.
+    */
+  includeDefaultFacets: optional boolean = true
 }
diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/search/AggregationMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/search/AggregationMetadata.pdl
index 0d3fa1e2a4ecf..d3d25083ff467 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/metadata/search/AggregationMetadata.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/search/AggregationMetadata.pdl
@@ -13,6 +13,11 @@ record AggregationMetadata {
      */
     displayName: optional string
 
+    /**
+     * Entity associated with this facet
+     */
+    entity: optional Urn
+
     /**
      * List of aggregations showing the number of documents falling into each bucket. e.g, for platform aggregation, the bucket can be hive, kafka, etc
      */
diff --git a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelGroupProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelGroupProperties.pdl
index b54e430038082..81c5e7a240f61 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelGroupProperties.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelGroupProperties.pdl
@@ -4,6 +4,7 @@ import com.linkedin.common.Urn
 import com.linkedin.common.Time
 import com.linkedin.common.VersionTag
 import com.linkedin.common.CustomProperties
+import com.linkedin.common.TimeStamp
 
 /**
  * Properties associated with an ML Model Group
@@ -13,6 +14,17 @@ import com.linkedin.common.CustomProperties
 }
 record MLModelGroupProperties includes CustomProperties {
 
+  /**
+   * Display name of the MLModelGroup
+   */
+  @Searchable = {
+    "fieldType": "WORD_GRAM",
+    "enableAutocomplete": true,
+    "boostScore": 10.0,
+    "queryByDefault": true,
+  }
+  name: optional string
+
   /**
    * Documentation of the MLModelGroup
    */
@@ -25,8 +37,31 @@ record MLModelGroupProperties includes CustomProperties {
   /**
    * Date when the MLModelGroup was developed
    */
+  @deprecated
   createdAt: optional Time
 
+  /**
+   * Time and Actor who created the MLModelGroup
+   */
+  created: optional TimeStamp
+
+  /**
+   * Date when the MLModelGroup was last modified
+   */
+  lastModified: optional TimeStamp
+
+  /**
+   * List of jobs (if any) used to train the model group. Visible in Lineage.
+   */
+  @Relationship = {
+    "/*": {
+      "name": "TrainedBy",
+      "entityTypes": [ "dataJob" ],
+      "isLineage": true
+    }
+  }
+  trainingJobs: optional array[Urn]
+
   /**
    * Version of the MLModelGroup
    */
diff --git a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelProperties.pdl
index 621a3e1747b50..d89d07384bba1 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelProperties.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelProperties.pdl
@@ -6,6 +6,7 @@ import com.linkedin.common.Time
 import com.linkedin.common.VersionTag
 import com.linkedin.common.CustomProperties
 import com.linkedin.common.ExternalReference
+import com.linkedin.common.TimeStamp
 
 /**
  * Properties associated with a ML Model
@@ -15,6 +16,18 @@ import com.linkedin.common.ExternalReference
 }
 record MLModelProperties includes CustomProperties, ExternalReference {
 
+  /**
+   * Display name of the MLModel
+   */
+  @Searchable = {
+    "fieldType": "WORD_GRAM",
+    "enableAutocomplete": true,
+    "boostScore": 10.0,
+    "queryByDefault": true,
+  }
+  name: optional string
+
+
   /**
    * Documentation of the MLModel
    */
@@ -27,8 +40,19 @@ record MLModelProperties includes CustomProperties, ExternalReference {
   /**
    * Date when the MLModel was developed
    */
+  @deprecated
   date: optional Time
 
+  /**
+   * Audit stamp containing who created this and when
+   */
+  created: optional TimeStamp
+
+  /**
+   * Date when the MLModel was last modified
+   */
+  lastModified: optional TimeStamp
+
   /**
    * Version of the MLModel
    */
@@ -93,12 +117,12 @@ record MLModelProperties includes CustomProperties, ExternalReference {
   deployments: optional array[Urn]
 
   /**
-   * List of jobs (if any) used to train the model
+   * List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.
    */
   @Relationship = {
     "/*": {
       "name": "TrainedBy",
-      "entityTypes": [ "dataJob" ],
+      "entityTypes": [ "dataJob", "dataProcessInstance" ],
       "isLineage": true
     }
   }
diff --git a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLTrainingRunProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLTrainingRunProperties.pdl
new file mode 100644
index 0000000000000..f8b8eeafe908b
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLTrainingRunProperties.pdl
@@ -0,0 +1,36 @@
+namespace com.linkedin.ml.metadata
+
+import com.linkedin.common.AuditStamp
+import com.linkedin.common.CustomProperties
+import com.linkedin.common.ExternalReference
+import com.linkedin.common.Urn
+import com.linkedin.common.JobFlowUrn
+import com.linkedin.common.DataJobUrn
+/**
+ * The inputs and outputs of this training run
+ */
+@Aspect = {
+  "name": "mlTrainingRunProperties",
+}
+record MLTrainingRunProperties includes CustomProperties, ExternalReference {
+
+  /**
+   * Run Id of the ML Training Run
+   */
+  id: optional string
+
+  /**
+   * List of URLs for the Outputs of the ML Training Run
+   */
+  outputUrls: optional array[string]
+
+  /**
+   * Hyperparameters of the ML Training Run
+   */
+  hyperParams: optional array[MLHyperParam]
+
+  /**
+   * Metrics of the ML Training Run
+   */
+  trainingMetrics: optional array[MLMetric]
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/platformresource/PlatformResourceType.pdl b/metadata-models/src/main/pegasus/com/linkedin/platformresource/PlatformResourceType.pdl
index 2f36eda9141ab..1a1dbea4359fb 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/platformresource/PlatformResourceType.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/platformresource/PlatformResourceType.pdl
@@ -9,9 +9,13 @@ enum PlatformResourceType {
     /**
      * e.g. a Slack member resource, Looker user resource, etc.
      */
-    USER_INFO,
+    USER_INFO, 
     /**
      * e.g. a Slack channel
      */
     CONVERSATION
+    /**
+      * e.g. Looker mapping of all user ids
+      */
+    USER_ID_MAPPING
 }
diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl
index 0b72d376b0be4..61731e8d37fd6 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl
@@ -16,7 +16,7 @@ record SchemaField {
   @Searchable = {
     "fieldName": "fieldPaths",
     "fieldType": "TEXT",
-    "boostScore": 5.0,
+    "boostScore": 1.0,
     "queryByDefault": "true"
   }
   fieldPath: SchemaFieldPath
diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl
index 3ddb2d2e571da..416e2c5c11e22 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl
@@ -89,25 +89,25 @@ record StructuredPropertyDefinition {
     version: optional string
 
     /**
-         * Created Audit stamp
-         */
-        @Searchable = {
-          "/time": {
-           "fieldName": "createdTime",
-           "fieldType": "DATETIME"
-          }
-        }
-        created: optional AuditStamp
+      * Created Audit stamp
+      */
+    @Searchable = {
+      "/time": {
+        "fieldName": "createdTime",
+        "fieldType": "DATETIME"
+      }
+    }
+    created: optional AuditStamp
 
-        /**
-         * Created Audit stamp
-         */
-        @Searchable = {
-          "/time": {
-           "fieldName": "lastModified",
-           "fieldType": "DATETIME"
-          }
-        }
-        lastModified: optional AuditStamp
+    /**
+     * Last Modified Audit stamp
+     */
+    @Searchable = {
+      "/time": {
+       "fieldName": "lastModified",
+       "fieldType": "DATETIME"
+      }
+    }
+    lastModified: optional AuditStamp
 }
 
diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl
new file mode 100644
index 0000000000000..fadcdfa5204e1
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl
@@ -0,0 +1,64 @@
+namespace com.linkedin.structured
+
+import com.linkedin.common.AuditStamp
+
+/**
+ * Settings specific to a structured property entity
+ */
+@Aspect = {
+   "name": "structuredPropertySettings"
+}
+record StructuredPropertySettings {
+  /**
+   * Whether or not this asset should be hidden in the main application
+   */
+  @Searchable = {
+    "fieldType": "BOOLEAN"
+  }
+  isHidden: boolean = false
+
+  /**
+   * Whether or not this asset should be displayed as a search filter
+   */
+  @Searchable = {
+    "fieldType": "BOOLEAN"
+  }
+  showInSearchFilters: boolean = false
+
+  /**
+   * Whether or not this asset should be displayed in the asset sidebar
+   */
+  @Searchable = {
+    "fieldType": "BOOLEAN"
+  }
+  showInAssetSummary: boolean = false
+
+  /**
+   * Whether or not this asset should be displayed as an asset badge on other
+   * asset's headers
+   */
+  @Searchable = {
+    "fieldType": "BOOLEAN"
+  }
+  showAsAssetBadge: boolean = false
+
+  /**
+   * Whether or not this asset should be displayed as a column in the schema field table
+   * in a Dataset's "Columns" tab.
+   */
+  @Searchable = {
+    "fieldType": "BOOLEAN"
+  }
+  showInColumnsTable: boolean = false
+
+  /**
+    * Last Modified Audit stamp
+    */
+  @Searchable = {
+    "/time": {
+      "fieldName": "lastModifiedSettings",
+      "fieldType": "DATETIME"
+    }
+  }
+  lastModified: optional AuditStamp
+}
diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml
index 9b692b51dc2b5..4fe170ced69f3 100644
--- a/metadata-models/src/main/resources/entity-registry.yml
+++ b/metadata-models/src/main/resources/entity-registry.yml
@@ -116,6 +116,10 @@ entities:
       - dataProcessInstanceRunEvent
       - status
       - testResults
+      - dataPlatformInstance
+      - subTypes
+      - container
+      - mlTrainingRunProperties
   - name: chart
     category: core
     keyAspect: chartKey
@@ -237,6 +241,7 @@ entities:
       - dataPlatformInstance
       - subTypes
       - ownership
+      - deprecation
       - container
       - globalTags
       - glossaryTerms
@@ -248,6 +253,7 @@ entities:
       - structuredProperties
       - forms
       - testResults
+      - access
   - name: tag
     category: core
     keyAspect: tagKey
@@ -600,6 +606,7 @@ entities:
     keyAspect: structuredPropertyKey
     aspects:
       - propertyDefinition
+      - structuredPropertySettings
       - institutionalMemory
       - status
   - name: form
@@ -673,6 +680,12 @@ plugins:
       supportedEntityAspectNames:
         - entityName: '*'
           aspectName: '*'
+    - className: 'com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator'
+      enabled: true
+      spring:
+        enabled: true
+      packageScan:
+        - com.linkedin.gms.factory.plugins
   mcpSideEffects:
     - className: 'com.linkedin.metadata.structuredproperties.hooks.PropertyDefinitionDeleteSideEffect'
       packageScan:
diff --git a/metadata-operation-context/build.gradle b/metadata-operation-context/build.gradle
index 650082ef0d25e..71b61528d187b 100644
--- a/metadata-operation-context/build.gradle
+++ b/metadata-operation-context/build.gradle
@@ -1,6 +1,7 @@
 plugins {
   id 'java-library'
 }
+apply from: '../gradle/coverage/java-coverage.gradle'
 
 dependencies {
   api project(':metadata-utils')
diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ActorContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ActorContext.java
index e65bf22991736..c08b7fad4dee3 100644
--- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ActorContext.java
+++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ActorContext.java
@@ -1,12 +1,23 @@
 package io.datahubproject.metadata.context;
 
+import static com.linkedin.metadata.Constants.CORP_USER_KEY_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.CORP_USER_STATUS_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.CORP_USER_STATUS_SUSPENDED;
+import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.SYSTEM_ACTOR;
+
 import com.datahub.authentication.Authentication;
+import com.linkedin.common.Status;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.entity.Aspect;
+import com.linkedin.identity.CorpUserStatus;
+import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.policy.DataHubPolicyInfo;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import lombok.Builder;
@@ -48,6 +59,43 @@ public Urn getActorUrn() {
     return UrnUtils.getUrn(authentication.getActor().toUrnStr());
   }
 
+  /**
+   * Actor is considered active if the user is not hard-deleted, soft-deleted, and is not suspended
+   *
+   * @param aspectRetriever aspect retriever - ideally the SystemEntityClient backed one for caching
+   * @return active status
+   */
+  public boolean isActive(AspectRetriever aspectRetriever) {
+    // system cannot be disabled
+    if (SYSTEM_ACTOR.equals(authentication.getActor().toUrnStr())) {
+      return true;
+    }
+
+    Urn selfUrn = UrnUtils.getUrn(authentication.getActor().toUrnStr());
+    Map<Urn, Map<String, Aspect>> urnAspectMap =
+        aspectRetriever.getLatestAspectObjects(
+            Set.of(selfUrn),
+            Set.of(STATUS_ASPECT_NAME, CORP_USER_STATUS_ASPECT_NAME, CORP_USER_KEY_ASPECT_NAME));
+
+    Map<String, Aspect> aspectMap = urnAspectMap.getOrDefault(selfUrn, Map.of());
+
+    if (!aspectMap.containsKey(CORP_USER_KEY_ASPECT_NAME)) {
+      // user is hard deleted
+      return false;
+    }
+
+    Status status =
+        Optional.ofNullable(aspectMap.get(STATUS_ASPECT_NAME))
+            .map(a -> new Status(a.data()))
+            .orElse(new Status().setRemoved(false));
+    CorpUserStatus corpUserStatus =
+        Optional.ofNullable(aspectMap.get(CORP_USER_STATUS_ASPECT_NAME))
+            .map(a -> new CorpUserStatus(a.data()))
+            .orElse(new CorpUserStatus().setStatus(""));
+
+    return !status.isRemoved() && !CORP_USER_STATUS_SUSPENDED.equals(corpUserStatus.getStatus());
+  }
+
   /**
    * The current implementation creates a cache entry unique for the set of policies.
    *
diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java
index 61bf40f54817e..9158129235b39 100644
--- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java
+++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java
@@ -16,6 +16,8 @@
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.utils.AuditStampUtils;
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
+import io.datahubproject.metadata.exception.ActorAccessException;
+import io.datahubproject.metadata.exception.OperationContextException;
 import java.util.Collection;
 import java.util.Objects;
 import java.util.Optional;
@@ -63,6 +65,24 @@ public static OperationContext asSession(
       @Nonnull Authorizer authorizer,
       @Nonnull Authentication sessionAuthentication,
       boolean allowSystemAuthentication) {
+    return OperationContext.asSession(
+        systemOperationContext,
+        requestContext,
+        authorizer,
+        sessionAuthentication,
+        allowSystemAuthentication,
+        false);
+  }
+
+  @Nonnull
+  public static OperationContext asSession(
+      OperationContext systemOperationContext,
+      @Nonnull RequestContext requestContext,
+      @Nonnull Authorizer authorizer,
+      @Nonnull Authentication sessionAuthentication,
+      boolean allowSystemAuthentication,
+      boolean skipCache)
+      throws ActorAccessException {
     return systemOperationContext.toBuilder()
         .operationContextConfig(
             // update allowed system authentication
@@ -71,7 +91,8 @@ public static OperationContext asSession(
                 .build())
         .authorizationContext(AuthorizationContext.builder().authorizer(authorizer).build())
         .requestContext(requestContext)
-        .build(sessionAuthentication);
+        .validationContext(systemOperationContext.getValidationContext())
+        .build(sessionAuthentication, skipCache);
   }
 
   /**
@@ -84,10 +105,14 @@ public static OperationContext asSession(
   public static OperationContext withSearchFlags(
       OperationContext opContext, Function<SearchFlags, SearchFlags> flagDefaults) {
 
-    return opContext.toBuilder()
-        // update search flags for the request's session
-        .searchContext(opContext.getSearchContext().withFlagDefaults(flagDefaults))
-        .build(opContext.getSessionActorContext());
+    try {
+      return opContext.toBuilder()
+          // update search flags for the request's session
+          .searchContext(opContext.getSearchContext().withFlagDefaults(flagDefaults))
+          .build(opContext.getSessionActorContext(), false);
+    } catch (OperationContextException e) {
+      throw new RuntimeException(e);
+    }
   }
 
   /**
@@ -100,10 +125,14 @@ public static OperationContext withSearchFlags(
   public static OperationContext withLineageFlags(
       OperationContext opContext, Function<LineageFlags, LineageFlags> flagDefaults) {
 
-    return opContext.toBuilder()
-        // update lineage flags for the request's session
-        .searchContext(opContext.getSearchContext().withLineageFlagDefaults(flagDefaults))
-        .build(opContext.getSessionActorContext());
+    try {
+      return opContext.toBuilder()
+          // update lineage flags for the request's session
+          .searchContext(opContext.getSearchContext().withLineageFlagDefaults(flagDefaults))
+          .build(opContext.getSessionActorContext(), false);
+    } catch (OperationContextException e) {
+      throw new RuntimeException(e);
+    }
   }
 
   /**
@@ -122,7 +151,8 @@ public static OperationContext asSystem(
       @Nonnull EntityRegistry entityRegistry,
       @Nullable ServicesRegistryContext servicesRegistryContext,
       @Nullable IndexConvention indexConvention,
-      @Nullable RetrieverContext retrieverContext) {
+      @Nullable RetrieverContext retrieverContext,
+      @Nonnull ValidationContext validationContext) {
     return asSystem(
         config,
         systemAuthentication,
@@ -130,6 +160,7 @@ public static OperationContext asSystem(
         servicesRegistryContext,
         indexConvention,
         retrieverContext,
+        validationContext,
         ObjectMapperContext.DEFAULT);
   }
 
@@ -140,6 +171,7 @@ public static OperationContext asSystem(
       @Nullable ServicesRegistryContext servicesRegistryContext,
       @Nullable IndexConvention indexConvention,
       @Nullable RetrieverContext retrieverContext,
+      @Nonnull ValidationContext validationContext,
       @Nonnull ObjectMapperContext objectMapperContext) {
 
     ActorContext systemActorContext =
@@ -151,17 +183,22 @@ public static OperationContext asSystem(
             ? SearchContext.EMPTY
             : SearchContext.builder().indexConvention(indexConvention).build();
 
-    return OperationContext.builder()
-        .operationContextConfig(systemConfig)
-        .systemActorContext(systemActorContext)
-        .searchContext(systemSearchContext)
-        .entityRegistryContext(EntityRegistryContext.builder().build(entityRegistry))
-        .servicesRegistryContext(servicesRegistryContext)
-        // Authorizer.EMPTY doesn't actually apply to system auth
-        .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
-        .retrieverContext(retrieverContext)
-        .objectMapperContext(objectMapperContext)
-        .build(systemAuthentication);
+    try {
+      return OperationContext.builder()
+          .operationContextConfig(systemConfig)
+          .systemActorContext(systemActorContext)
+          .searchContext(systemSearchContext)
+          .entityRegistryContext(EntityRegistryContext.builder().build(entityRegistry))
+          .servicesRegistryContext(servicesRegistryContext)
+          // Authorizer.EMPTY doesn't actually apply to system auth
+          .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
+          .retrieverContext(retrieverContext)
+          .objectMapperContext(objectMapperContext)
+          .validationContext(validationContext)
+          .build(systemAuthentication, false);
+    } catch (OperationContextException e) {
+      throw new RuntimeException(e);
+    }
   }
 
   @Nonnull private final OperationContextConfig operationContextConfig;
@@ -172,8 +209,9 @@ public static OperationContext asSystem(
   @Nonnull private final EntityRegistryContext entityRegistryContext;
   @Nullable private final ServicesRegistryContext servicesRegistryContext;
   @Nullable private final RequestContext requestContext;
-  @Nullable private final RetrieverContext retrieverContext;
+  @Nonnull private final RetrieverContext retrieverContext;
   @Nonnull private final ObjectMapperContext objectMapperContext;
+  @Nonnull private final ValidationContext validationContext;
 
   public OperationContext withSearchFlags(
       @Nonnull Function<SearchFlags, SearchFlags> flagDefaults) {
@@ -188,13 +226,15 @@ public OperationContext withLineageFlags(
   public OperationContext asSession(
       @Nonnull RequestContext requestContext,
       @Nonnull Authorizer authorizer,
-      @Nonnull Authentication sessionAuthentication) {
+      @Nonnull Authentication sessionAuthentication)
+      throws ActorAccessException {
     return OperationContext.asSession(
         this,
         requestContext,
         authorizer,
         sessionAuthentication,
-        getOperationContextConfig().isAllowSystemAuthentication());
+        getOperationContextConfig().isAllowSystemAuthentication(),
+        false);
   }
 
   @Nonnull
@@ -278,17 +318,9 @@ public AuditStamp getAuditStamp() {
     return getAuditStamp(null);
   }
 
-  public Optional<RetrieverContext> getRetrieverContext() {
-    return Optional.ofNullable(retrieverContext);
-  }
-
-  @Nullable
+  @Nonnull
   public AspectRetriever getAspectRetriever() {
-    return getAspectRetrieverOpt().orElse(null);
-  }
-
-  public Optional<AspectRetriever> getAspectRetrieverOpt() {
-    return getRetrieverContext().map(RetrieverContext::getAspectRetriever);
+    return retrieverContext.getAspectRetriever();
   }
 
   /**
@@ -330,10 +362,7 @@ public String getGlobalContextId() {
                     ? EmptyContext.EMPTY
                     : getServicesRegistryContext())
             .add(getRequestContext() == null ? EmptyContext.EMPTY : getRequestContext())
-            .add(
-                getRetrieverContext().isPresent()
-                    ? getRetrieverContext().get()
-                    : EmptyContext.EMPTY)
+            .add(getRetrieverContext())
             .add(getObjectMapperContext())
             .build()
             .stream()
@@ -358,10 +387,7 @@ public String getSearchContextId() {
                 getServicesRegistryContext() == null
                     ? EmptyContext.EMPTY
                     : getServicesRegistryContext())
-            .add(
-                getRetrieverContext().isPresent()
-                    ? getRetrieverContext().get()
-                    : EmptyContext.EMPTY)
+            .add(getRetrieverContext())
             .build()
             .stream()
             .map(ContextInterface::getCacheKeyComponent)
@@ -432,6 +458,12 @@ public static class OperationContextBuilder {
 
     @Nonnull
     public OperationContext build(@Nonnull Authentication sessionAuthentication) {
+      return build(sessionAuthentication, false);
+    }
+
+    @Nonnull
+    public OperationContext build(
+        @Nonnull Authentication sessionAuthentication, boolean skipCache) {
       final Urn actorUrn = UrnUtils.getUrn(sessionAuthentication.getActor().toUrnStr());
       final ActorContext sessionActor =
           ActorContext.builder()
@@ -445,11 +477,20 @@ public OperationContext build(@Nonnull Authentication sessionAuthentication) {
               .policyInfoSet(this.authorizationContext.getAuthorizer().getActorPolicies(actorUrn))
               .groupMembership(this.authorizationContext.getAuthorizer().getActorGroups(actorUrn))
               .build();
-      return build(sessionActor);
+      return build(sessionActor, skipCache);
     }
 
     @Nonnull
-    public OperationContext build(@Nonnull ActorContext sessionActor) {
+    public OperationContext build(@Nonnull ActorContext sessionActor, boolean skipCache) {
+      AspectRetriever retriever =
+          skipCache
+              ? this.retrieverContext.getAspectRetriever()
+              : this.retrieverContext.getCachingAspectRetriever();
+
+      if (!sessionActor.isActive(retriever)) {
+        throw new ActorAccessException("Actor is not active");
+      }
+
       return new OperationContext(
           this.operationContextConfig,
           sessionActor,
@@ -460,9 +501,8 @@ public OperationContext build(@Nonnull ActorContext sessionActor) {
           this.servicesRegistryContext,
           this.requestContext,
           this.retrieverContext,
-          this.objectMapperContext != null
-              ? this.objectMapperContext
-              : ObjectMapperContext.DEFAULT);
+          this.objectMapperContext != null ? this.objectMapperContext : ObjectMapperContext.DEFAULT,
+          this.validationContext);
     }
 
     private OperationContext build() {
diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RetrieverContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RetrieverContext.java
index 9337fbfe3bb00..9afc4138810bb 100644
--- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RetrieverContext.java
+++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RetrieverContext.java
@@ -1,8 +1,10 @@
 package io.datahubproject.metadata.context;
 
 import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.entity.SearchRetriever;
+import java.util.Objects;
 import java.util.Optional;
 import javax.annotation.Nonnull;
 import lombok.Builder;
@@ -15,10 +17,37 @@ public class RetrieverContext
 
   @Nonnull private final GraphRetriever graphRetriever;
   @Nonnull private final AspectRetriever aspectRetriever;
+  @Nonnull private final CachingAspectRetriever cachingAspectRetriever;
   @Nonnull private final SearchRetriever searchRetriever;
 
   @Override
   public Optional<Integer> getCacheKeyComponent() {
     return Optional.empty();
   }
+
+  public static class RetrieverContextBuilder {
+    public RetrieverContext build() {
+      if (this.aspectRetriever == null && this.cachingAspectRetriever != null) {
+        this.aspectRetriever = this.cachingAspectRetriever;
+      }
+
+      if (this.cachingAspectRetriever == null
+          && this.aspectRetriever instanceof CachingAspectRetriever) {
+        this.cachingAspectRetriever = (CachingAspectRetriever) this.aspectRetriever;
+      }
+
+      return new RetrieverContext(
+          this.graphRetriever,
+          Objects.requireNonNull(this.aspectRetriever),
+          Objects.requireNonNull(this.cachingAspectRetriever),
+          this.searchRetriever);
+    }
+  }
+
+  public static final RetrieverContext EMPTY =
+      RetrieverContext.builder()
+          .graphRetriever(GraphRetriever.EMPTY)
+          .searchRetriever(SearchRetriever.EMPTY)
+          .cachingAspectRetriever(CachingAspectRetriever.EMPTY)
+          .build();
 }
diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ValidationContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ValidationContext.java
new file mode 100644
index 0000000000000..76560f087f22d
--- /dev/null
+++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ValidationContext.java
@@ -0,0 +1,18 @@
+package io.datahubproject.metadata.context;
+
+import java.util.Optional;
+import lombok.Builder;
+import lombok.Getter;
+
+/** Context holder for environment variables relevant to operations */
+@Builder
+@Getter
+public class ValidationContext implements ContextInterface {
+  // Uses alternate validation flow for MCP ingestion
+  private final boolean alternateValidation;
+
+  @Override
+  public Optional<Integer> getCacheKeyComponent() {
+    return Optional.of(alternateValidation ? 1 : 0);
+  }
+}
diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/exception/ActorAccessException.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/exception/ActorAccessException.java
new file mode 100644
index 0000000000000..bca2594b96430
--- /dev/null
+++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/exception/ActorAccessException.java
@@ -0,0 +1,7 @@
+package io.datahubproject.metadata.exception;
+
+public class ActorAccessException extends OperationContextException {
+  public ActorAccessException(String string) {
+    super(string);
+  }
+}
diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/exception/OperationContextException.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/exception/OperationContextException.java
new file mode 100644
index 0000000000000..1aac8dc3e60ec
--- /dev/null
+++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/exception/OperationContextException.java
@@ -0,0 +1,9 @@
+package io.datahubproject.metadata.exception;
+
+public class OperationContextException extends RuntimeException {
+  public OperationContextException(String message) {
+    super(message);
+  }
+
+  public OperationContextException() {}
+}
diff --git a/metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java b/metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java
index cdcbb540eeda4..4abfbb196f067 100644
--- a/metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java
+++ b/metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java
@@ -8,21 +8,17 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
 import com.linkedin.entity.Aspect;
+import com.linkedin.identity.CorpUserInfo;
+import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.aspect.GraphRetriever;
-import com.linkedin.metadata.aspect.SystemAspect;
-import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult;
 import com.linkedin.metadata.entity.SearchRetriever;
 import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistryException;
 import com.linkedin.metadata.models.registry.MergedEntityRegistry;
 import com.linkedin.metadata.models.registry.SnapshotEntityRegistry;
-import com.linkedin.metadata.query.filter.Filter;
-import com.linkedin.metadata.query.filter.RelationshipFilter;
-import com.linkedin.metadata.query.filter.SortCriterion;
-import com.linkedin.metadata.search.ScrollResult;
-import com.linkedin.metadata.search.SearchEntityArray;
 import com.linkedin.metadata.snapshot.Snapshot;
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
 import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl;
@@ -31,15 +27,15 @@
 import io.datahubproject.metadata.context.RequestContext;
 import io.datahubproject.metadata.context.RetrieverContext;
 import io.datahubproject.metadata.context.ServicesRegistryContext;
-import java.util.List;
+import io.datahubproject.metadata.context.ValidationContext;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.Builder;
 
 /**
  * Useful for testing. If the defaults are not sufficient, try using the .toBuilder() and replacing
@@ -58,6 +54,8 @@ public class TestOperationContexts {
           .build();
 
   private static EntityRegistry defaultEntityRegistryInstance;
+  private static ValidationContext defaultValidationContext =
+      ValidationContext.builder().alternateValidation(false).build();
 
   public static EntityRegistry defaultEntityRegistry() {
     if (defaultEntityRegistryInstance == null) {
@@ -78,26 +76,53 @@ public static EntityRegistry defaultEntityRegistry() {
     return defaultEntityRegistryInstance;
   }
 
-  public static AspectRetriever emptyAspectRetriever(
+  public static RetrieverContext emptyActiveUsersRetrieverContext(
       @Nullable Supplier<EntityRegistry> entityRegistrySupplier) {
-    return new EmptyAspectRetriever(
-        () ->
-            Optional.ofNullable(entityRegistrySupplier)
-                .map(Supplier::get)
-                .orElse(defaultEntityRegistry()));
-  }
 
-  public static GraphRetriever emptyGraphRetriever = new EmptyGraphRetriever();
-  public static SearchRetriever emptySearchRetriever = new EmptySearchRetriever();
+    return RetrieverContext.builder()
+        .cachingAspectRetriever(emptyActiveUsersAspectRetriever(entityRegistrySupplier))
+        .graphRetriever(GraphRetriever.EMPTY)
+        .searchRetriever(SearchRetriever.EMPTY)
+        .build();
+  }
 
-  public static RetrieverContext emptyRetrieverContext(
+  public static CachingAspectRetriever emptyActiveUsersAspectRetriever(
       @Nullable Supplier<EntityRegistry> entityRegistrySupplier) {
 
-    return RetrieverContext.builder()
-        .aspectRetriever(emptyAspectRetriever(entityRegistrySupplier))
-        .graphRetriever(emptyGraphRetriever)
-        .searchRetriever(emptySearchRetriever)
-        .build();
+    return new CachingAspectRetriever.EmptyAspectRetriever() {
+
+      @Nonnull
+      @Override
+      public Map<Urn, Map<String, Aspect>> getLatestAspectObjects(
+          Set<Urn> urns, Set<String> aspectNames) {
+        if (urns.stream().allMatch(urn -> urn.toString().startsWith("urn:li:corpuser:"))
+            && aspectNames.contains(Constants.CORP_USER_KEY_ASPECT_NAME)) {
+          return urns.stream()
+              .map(
+                  urn ->
+                      Map.entry(
+                          urn,
+                          Map.of(
+                              Constants.CORP_USER_KEY_ASPECT_NAME,
+                              new Aspect(
+                                  new CorpUserInfo()
+                                      .setActive(true)
+                                      .setEmail(urn.getId())
+                                      .setDisplayName(urn.getId())
+                                      .data()))))
+              .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        }
+        return super.getLatestAspectObjects(urns, aspectNames);
+      }
+
+      @Nonnull
+      @Override
+      public EntityRegistry getEntityRegistry() {
+        return Optional.ofNullable(entityRegistrySupplier)
+            .map(Supplier::get)
+            .orElse(defaultEntityRegistry());
+      }
+    };
   }
 
   public static OperationContext systemContextNoSearchAuthorization(
@@ -114,6 +139,11 @@ public static OperationContext systemContextNoSearchAuthorization() {
     return systemContextNoSearchAuthorization(null, null, null);
   }
 
+  public static OperationContext systemContextNoValidate() {
+    return systemContextNoSearchAuthorization(
+        null, null, null, () -> ValidationContext.builder().alternateValidation(true).build());
+  }
+
   public static OperationContext systemContextNoSearchAuthorization(
       @Nullable EntityRegistry entityRegistry, @Nullable IndexConvention indexConvention) {
     return systemContextNoSearchAuthorization(() -> entityRegistry, null, () -> indexConvention);
@@ -132,8 +162,10 @@ public static OperationContext systemContextNoSearchAuthorization(
     RetrieverContext retrieverContext =
         RetrieverContext.builder()
             .aspectRetriever(aspectRetriever)
-            .graphRetriever(emptyGraphRetriever)
-            .searchRetriever(emptySearchRetriever)
+            .cachingAspectRetriever(
+                emptyActiveUsersAspectRetriever(() -> aspectRetriever.getEntityRegistry()))
+            .graphRetriever(GraphRetriever.EMPTY)
+            .searchRetriever(SearchRetriever.EMPTY)
             .build();
     return systemContextNoSearchAuthorization(
         () -> retrieverContext.getAspectRetriever().getEntityRegistry(),
@@ -160,9 +192,27 @@ public static OperationContext systemContextNoSearchAuthorization(
         entityRegistrySupplier,
         retrieverContextSupplier,
         indexConventionSupplier,
+        null,
         null);
   }
 
+  public static OperationContext systemContextNoSearchAuthorization(
+      @Nullable Supplier<EntityRegistry> entityRegistrySupplier,
+      @Nullable Supplier<RetrieverContext> retrieverContextSupplier,
+      @Nullable Supplier<IndexConvention> indexConventionSupplier,
+      @Nullable Supplier<ValidationContext> environmentContextSupplier) {
+
+    return systemContext(
+        null,
+        null,
+        null,
+        entityRegistrySupplier,
+        retrieverContextSupplier,
+        indexConventionSupplier,
+        null,
+        environmentContextSupplier);
+  }
+
   public static OperationContext systemContext(
       @Nullable Supplier<OperationContextConfig> configSupplier,
       @Nullable Supplier<Authentication> systemAuthSupplier,
@@ -170,7 +220,8 @@ public static OperationContext systemContext(
       @Nullable Supplier<EntityRegistry> entityRegistrySupplier,
       @Nullable Supplier<RetrieverContext> retrieverContextSupplier,
       @Nullable Supplier<IndexConvention> indexConventionSupplier,
-      @Nullable Consumer<OperationContext> postConstruct) {
+      @Nullable Consumer<OperationContext> postConstruct,
+      @Nullable Supplier<ValidationContext> environmentContextSupplier) {
 
     OperationContextConfig config =
         Optional.ofNullable(configSupplier).map(Supplier::get).orElse(DEFAULT_OPCONTEXT_CONFIG);
@@ -181,7 +232,7 @@ public static OperationContext systemContext(
     RetrieverContext retrieverContext =
         Optional.ofNullable(retrieverContextSupplier)
             .map(Supplier::get)
-            .orElse(emptyRetrieverContext(entityRegistrySupplier));
+            .orElse(emptyActiveUsersRetrieverContext(entityRegistrySupplier));
 
     EntityRegistry entityRegistry =
         Optional.ofNullable(entityRegistrySupplier)
@@ -196,6 +247,11 @@ public static OperationContext systemContext(
     ServicesRegistryContext servicesRegistryContext =
         Optional.ofNullable(servicesRegistrySupplier).orElse(() -> null).get();
 
+    ValidationContext validationContext =
+        Optional.ofNullable(environmentContextSupplier)
+            .map(Supplier::get)
+            .orElse(defaultValidationContext);
+
     OperationContext operationContext =
         OperationContext.asSystem(
             config,
@@ -203,7 +259,8 @@ public static OperationContext systemContext(
             entityRegistry,
             servicesRegistryContext,
             indexConvention,
-            retrieverContext);
+            retrieverContext,
+            validationContext);
 
     if (postConstruct != null) {
       postConstruct.accept(operationContext);
@@ -265,66 +322,5 @@ public static OperationContext userContextNoSearchAuthorization(
         .asSession(requestContext, Authorizer.EMPTY, TEST_USER_AUTH);
   }
 
-  @Builder
-  public static class EmptyAspectRetriever implements AspectRetriever {
-    private final Supplier<EntityRegistry> entityRegistrySupplier;
-
-    @Nonnull
-    @Override
-    public Map<Urn, Map<String, Aspect>> getLatestAspectObjects(
-        Set<Urn> urns, Set<String> aspectNames) {
-      return Map.of();
-    }
-
-    @Nonnull
-    @Override
-    public Map<Urn, Map<String, SystemAspect>> getLatestSystemAspects(
-        Map<Urn, Set<String>> urnAspectNames) {
-      return Map.of();
-    }
-
-    @Nonnull
-    @Override
-    public EntityRegistry getEntityRegistry() {
-      return entityRegistrySupplier.get();
-    }
-  }
-
-  public static class EmptyGraphRetriever implements GraphRetriever {
-
-    @Nonnull
-    @Override
-    public RelatedEntitiesScrollResult scrollRelatedEntities(
-        @Nullable List<String> sourceTypes,
-        @Nonnull Filter sourceEntityFilter,
-        @Nullable List<String> destinationTypes,
-        @Nonnull Filter destinationEntityFilter,
-        @Nonnull List<String> relationshipTypes,
-        @Nonnull RelationshipFilter relationshipFilter,
-        @Nonnull List<SortCriterion> sortCriterion,
-        @Nullable String scrollId,
-        int count,
-        @Nullable Long startTimeMillis,
-        @Nullable Long endTimeMillis) {
-      return new RelatedEntitiesScrollResult(0, 0, null, List.of());
-    }
-  }
-
-  public static class EmptySearchRetriever implements SearchRetriever {
-
-    @Override
-    public ScrollResult scroll(
-        @Nonnull List<String> entities,
-        @Nullable Filter filters,
-        @Nullable String scrollId,
-        int count) {
-      ScrollResult empty = new ScrollResult();
-      empty.setEntities(new SearchEntityArray());
-      empty.setNumEntities(0);
-      empty.setPageSize(0);
-      return empty;
-    }
-  }
-
   private TestOperationContexts() {}
 }
diff --git a/metadata-operation-context/src/test/java/io/datahubproject/metadata/context/OperationContextTest.java b/metadata-operation-context/src/test/java/io/datahubproject/metadata/context/OperationContextTest.java
index 6a5e9e04f7dda..f77b244d8f2d8 100644
--- a/metadata-operation-context/src/test/java/io/datahubproject/metadata/context/OperationContextTest.java
+++ b/metadata-operation-context/src/test/java/io/datahubproject/metadata/context/OperationContextTest.java
@@ -8,6 +8,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.linkedin.metadata.models.registry.EntityRegistry;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
 import org.testng.annotations.Test;
 
 public class OperationContextTest {
@@ -25,7 +26,8 @@ public void testSystemPrivilegeEscalation() {
             mock(EntityRegistry.class),
             mock(ServicesRegistryContext.class),
             null,
-            mock(RetrieverContext.class));
+            TestOperationContexts.emptyActiveUsersRetrieverContext(null),
+            mock(ValidationContext.class));
 
     OperationContext opContext =
         systemOpContext.asSession(RequestContext.TEST, Authorizer.EMPTY, userAuth);
diff --git a/metadata-service/auth-config/build.gradle b/metadata-service/auth-config/build.gradle
index 8302e3b0c2fe6..de787e9f350dc 100644
--- a/metadata-service/auth-config/build.gradle
+++ b/metadata-service/auth-config/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(path: ':metadata-models')
   implementation project(path: ':metadata-auth:auth-api')
diff --git a/metadata-service/auth-filter/build.gradle b/metadata-service/auth-filter/build.gradle
index 9d763ca11421b..5e73246317677 100644
--- a/metadata-service/auth-filter/build.gradle
+++ b/metadata-service/auth-filter/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(':metadata-auth:auth-api')
   implementation project(':metadata-service:auth-impl')
diff --git a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java
index 0a54677eb6149..30f98180f8018 100644
--- a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java
+++ b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java
@@ -98,11 +98,12 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha
     }
 
     if (authentication != null) {
+      String actorUrnStr = authentication.getActor().toUrnStr();
       // Successfully authenticated.
       log.debug(
-          String.format(
-              "Successfully authenticated request for Actor with type: %s, id: %s",
-              authentication.getActor().getType(), authentication.getActor().getId()));
+          "Successfully authenticated request for Actor with type: {}, id: {}",
+          authentication.getActor().getType(),
+          authentication.getActor().getId());
       AuthenticationContext.setAuthentication(authentication);
       chain.doFilter(request, response);
     } else {
@@ -110,7 +111,9 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha
       log.debug(
           "Failed to authenticate request. Received 'null' Authentication value from authenticator chain.");
       ((HttpServletResponse) response)
-          .sendError(HttpServletResponse.SC_UNAUTHORIZED, "Unauthorized to perform this action.");
+          .sendError(
+              HttpServletResponse.SC_UNAUTHORIZED,
+              "Unauthorized to perform this action due to expired auth.");
       return;
     }
     AuthenticationContext.remove();
diff --git a/metadata-service/auth-impl/build.gradle b/metadata-service/auth-impl/build.gradle
index 4f4b0658caf24..b5c20ccabe6d2 100644
--- a/metadata-service/auth-impl/build.gradle
+++ b/metadata-service/auth-impl/build.gradle
@@ -2,6 +2,7 @@ plugins {
   id 'java'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
 
 compileJava {
   options.debug = true
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java
index 6724f35d840ad..a9871f1ed9948 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java
@@ -145,7 +145,7 @@ public String generateAccessToken(
     _entityService.ingestProposal(
         systemOperationContext,
         AspectsBatchImpl.builder()
-            .mcps(List.of(proposal), auditStamp, systemOperationContext.getRetrieverContext().get())
+            .mcps(List.of(proposal), auditStamp, systemOperationContext.getRetrieverContext())
             .build(),
         false);
 
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
index 985cfe48f6bcf..4437682bfeb0a 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
@@ -54,6 +54,7 @@
 import io.datahubproject.metadata.context.OperationContextConfig;
 import io.datahubproject.metadata.context.RetrieverContext;
 import io.datahubproject.metadata.context.ServicesRegistryContext;
+import io.datahubproject.metadata.context.ValidationContext;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -318,7 +319,8 @@ public void setupTest() throws Exception {
             mock(EntityRegistry.class),
             mock(ServicesRegistryContext.class),
             mock(IndexConvention.class),
-            mock(RetrieverContext.class));
+            mock(RetrieverContext.class),
+            mock(ValidationContext.class));
 
     _dataHubAuthorizer =
         new DataHubAuthorizer(
@@ -598,7 +600,6 @@ private DataHubPolicyInfo createDataHubPolicyInfoFor(
     dataHubPolicyInfo.setDisplayName("My Test Display");
     dataHubPolicyInfo.setDescription("My test display!");
     dataHubPolicyInfo.setEditable(true);
-
     dataHubPolicyInfo.setActors(actorFilter);
 
     final DataHubResourceFilter resourceFilter = new DataHubResourceFilter();
diff --git a/metadata-service/auth-servlet-impl/build.gradle b/metadata-service/auth-servlet-impl/build.gradle
index 29e452472358b..21fe63778cbeb 100644
--- a/metadata-service/auth-servlet-impl/build.gradle
+++ b/metadata-service/auth-servlet-impl/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(':metadata-auth:auth-api')
   implementation project(':metadata-service:auth-impl')
diff --git a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java
index de2582af00a93..5d4542cf0826e 100644
--- a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java
+++ b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java
@@ -138,7 +138,9 @@ CompletableFuture<ResponseEntity<String>> generateSessionTokenForUser(
     }
 
     log.info("Attempting to generate session token for user {}", userId.asText());
-    final String actorId = AuthenticationContext.getAuthentication().getActor().getId();
+    Authentication authentication = AuthenticationContext.getAuthentication();
+    final String actorId = authentication.getActor().getId();
+    final String actorUrn = authentication.getActor().toUrnStr();
     return CompletableFuture.supplyAsync(
         () -> {
           // 1. Verify that only those authorized to generate a token (datahub system) are able to.
@@ -164,7 +166,7 @@ CompletableFuture<ResponseEntity<String>> generateSessionTokenForUser(
           }
           throw HttpClientErrorException.create(
               HttpStatus.UNAUTHORIZED,
-              "Unauthorized to perform this action.",
+              actorUrn + " unauthorized to perform this action.",
               new HttpHeaders(),
               null,
               null);
diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle
index f912e2ac01f0b..3c7e38b004fae 100644
--- a/metadata-service/configuration/build.gradle
+++ b/metadata-service/configuration/build.gradle
@@ -1,6 +1,8 @@
 plugins {
     id 'java-library'
 }
+
+apply from: '../../gradle/coverage/java-coverage.gradle'
 apply from: "../../gradle/versioning/versioning.gradle"
 
 dependencies {
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/metadata-service/configuration/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java
index 167515a13c4da..28abb26be1f52 100644
--- a/metadata-service/configuration/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java
+++ b/metadata-service/configuration/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java
@@ -23,4 +23,6 @@ public class FeatureFlags {
   private boolean dataContractsEnabled = false;
   private boolean editableDatasetNameEnabled = false;
   private boolean showSeparateSiblings = false;
+  private boolean alternateMCPValidation = false;
+  private boolean showManageStructuredProperties = false;
 }
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubAppConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubAppConfiguration.java
index cc96429c65e76..fb7a5103952af 100644
--- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubAppConfiguration.java
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubAppConfiguration.java
@@ -58,4 +58,7 @@ public class DataHubAppConfiguration {
 
   /** MCP throttling configuration */
   private MetadataChangeProposalConfig metadataChangeProposal;
+
+  /** Timeseries Aspect Service configuration */
+  private TimeseriesAspectServiceConfig timeseriesAspectService;
 }
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/EbeanConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/EbeanConfiguration.java
index 47b406e695a3f..6eb31e14a2d3b 100644
--- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/EbeanConfiguration.java
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/EbeanConfiguration.java
@@ -23,6 +23,7 @@ public class EbeanConfiguration {
   private boolean autoCreateDdl;
   private boolean postgresUseIamAuth;
   private LockingConfiguration locking;
+  private String batchGetMethod;
 
   public static final EbeanConfiguration testDefault =
       EbeanConfiguration.builder().locking(LockingConfiguration.testDefault).build();
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ExecutorServiceConfig.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ExecutorServiceConfig.java
new file mode 100644
index 0000000000000..d319ce213e303
--- /dev/null
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ExecutorServiceConfig.java
@@ -0,0 +1,16 @@
+package com.linkedin.metadata.config;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder(toBuilder = true)
+@AllArgsConstructor
+@NoArgsConstructor
+public class ExecutorServiceConfig {
+  @Builder.Default private int concurrency = 2;
+  @Builder.Default private int queueSize = 100;
+  @Builder.Default private int keepAlive = 60;
+}
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MCPValidationConfig.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MCPValidationConfig.java
new file mode 100644
index 0000000000000..622dbb010a576
--- /dev/null
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MCPValidationConfig.java
@@ -0,0 +1,9 @@
+package com.linkedin.metadata.config;
+
+import com.linkedin.metadata.config.structuredProperties.extensions.ModelExtensionValidationConfiguration;
+import lombok.Data;
+
+@Data
+public class MCPValidationConfig {
+  private ModelExtensionValidationConfiguration extensions;
+}
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MetadataChangeProposalConfig.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MetadataChangeProposalConfig.java
index 4e8c18912c40e..86b4a1b8562b7 100644
--- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MetadataChangeProposalConfig.java
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MetadataChangeProposalConfig.java
@@ -8,6 +8,7 @@
 public class MetadataChangeProposalConfig {
 
   ThrottlesConfig throttle;
+  MCPValidationConfig validation;
   SideEffectsConfig sideEffects;
 
   @Data
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TimeseriesAspectServiceConfig.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TimeseriesAspectServiceConfig.java
new file mode 100644
index 0000000000000..3708e05d5cb67
--- /dev/null
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TimeseriesAspectServiceConfig.java
@@ -0,0 +1,14 @@
+package com.linkedin.metadata.config;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder(toBuilder = true)
+@AllArgsConstructor
+@NoArgsConstructor
+public class TimeseriesAspectServiceConfig {
+  @Builder.Default private ExecutorServiceConfig query = ExecutorServiceConfig.builder().build();
+}
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java
index 60f3e1b4fef76..9b476483a2baf 100644
--- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java
@@ -8,4 +8,13 @@ public class ConsumerConfiguration {
   private int maxPartitionFetchBytes;
   private boolean stopOnDeserializationError;
   private boolean healthCheckEnabled;
+
+  private ConsumerOptions mcp;
+  private ConsumerOptions mcl;
+  private ConsumerOptions pe;
+
+  @Data
+  public static class ConsumerOptions {
+    private String autoOffsetReset;
+  }
 }
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java
index b03aedc1a7b5e..ae0d3a3bb4647 100644
--- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java
@@ -20,6 +20,10 @@ public class KafkaConfiguration {
       "spring.deserializer.key.delegate.class";
   public static final String VALUE_DESERIALIZER_DELEGATE_CLASS =
       "spring.deserializer.value.delegate.class";
+  public static final String MCP_EVENT_CONSUMER_NAME = "mcpEventConsumer";
+  public static final String MCL_EVENT_CONSUMER_NAME = "mclEventConsumer";
+  public static final String PE_EVENT_CONSUMER_NAME = "platformEventConsumer";
+  public static final String DEFAULT_EVENT_CONSUMER_NAME = "kafkaEventConsumer";
 
   private String bootstrapServers;
 
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/structuredProperties/extensions/ModelExtensionValidationConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/structuredProperties/extensions/ModelExtensionValidationConfiguration.java
new file mode 100644
index 0000000000000..71db309154038
--- /dev/null
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/structuredProperties/extensions/ModelExtensionValidationConfiguration.java
@@ -0,0 +1,10 @@
+package com.linkedin.metadata.config.structuredProperties.extensions;
+
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
+@Data
+@Slf4j
+public class ModelExtensionValidationConfiguration {
+  private boolean enabled;
+}
diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml
index ef3ae76d81fae..b997bc108e4ba 100644
--- a/metadata-service/configuration/src/main/resources/application.yaml
+++ b/metadata-service/configuration/src/main/resources/application.yaml
@@ -118,6 +118,12 @@ searchService:
       pageSize: ${SEARCH_SERVICE_FILTER_DOMAIN_EXPANSION_PAGE_SIZE:100}
       limit: ${SEARCH_SERVICE_FILTER_DOMAIN_EXPANSION_LIMIT:100}
 
+timeseriesAspectService:
+  query:
+    concurrency: ${TIMESERIES_ASPECT_SERVICE_QUERY_CONCURRENCY:10} # parallel threads
+    queueSize: ${TIMESERIES_ASPECT_SERVICE_QUERY)QUEUE_SIZE:500}
+    threadKeepAlive: ${TIMESERIES_ASPECT_SERVICE_QUERY_THREAD_KEEP_ALIVE:60}
+
 configEntityRegistry:
   path: ${ENTITY_REGISTRY_CONFIG_PATH:../../metadata-models/src/main/resources/entity-registry.yml}
   # Priority is given to the `path` setting above (outside jar)
@@ -158,8 +164,9 @@ ebean:
   waitTimeoutMillis: ${EBEAN_WAIT_TIMEOUT_MILLIS:1000}
   autoCreateDdl: ${EBEAN_AUTOCREATE:false}
   postgresUseIamAuth: ${EBEAN_POSTGRES_USE_AWS_IAM_AUTH:false}
+  batchGetMethod: ${EBEAN_BATCH_GET_METHOD:IN} # Alternative UNION
   locking:
-    enabled: ${EBEAN_LOCKING_ENABLED:true}
+    enabled: ${EBEAN_LOCKING_ENABLED:false}
     durationSeconds: ${EBEAN_LOCKING_DURATION_SECONDS:60}
     maximumLocks: ${EBEAN_LOCKING_MAXIMUM_LOCKS:20000}
 
@@ -289,6 +296,13 @@ kafka:
     maxPartitionFetchBytes: ${KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES:5242880} # the max bytes consumed per partition
     stopOnDeserializationError: ${KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR:true} # Stops kafka listener container on deserialization error, allows user to fix problems before moving past problematic offset. If false will log and move forward past the offset
     healthCheckEnabled: ${KAFKA_CONSUMER_HEALTH_CHECK_ENABLED:true} # Sets the health indicator to down when a message listener container has stopped due to a deserialization failure, will force consumer apps to restart through k8s and docker-compose health mechanisms
+    mcp:
+      autoOffsetReset: ${KAFKA_CONSUMER_MCP_AUTO_OFFSET_RESET:earliest}
+    mcl:
+      autoOffsetReset: ${KAFKA_CONSUMER_MCL_AUTO_OFFSET_RESET:earliest}
+    pe:
+      autoOffsetReset: ${KAFKA_CONSUMER_PE_AUTO_OFFSET_RESET:latest}
+
   schemaRegistry:
     type: ${SCHEMA_REGISTRY_TYPE:KAFKA} # INTERNAL or KAFKA or AWS_GLUE
     url: ${KAFKA_SCHEMAREGISTRY_URL:http://localhost:8081}
@@ -372,6 +386,11 @@ systemUpdate:
     batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_BROWSE_PATHS_V2_BATCH_SIZE:5000}
     reprocess:
       enabled: ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false}
+  ingestionIndices:
+    enabled: ${BOOTSTRAP_SYSTEM_UPDATE_INGESTION_INDICES_ENABLED:true}
+    batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_INGESTION_INDICES_BATCH_SIZE:5000}
+    delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_INGESTION_INDICES_DELAY_MS:1000}
+    limit: ${BOOTSTRAP_SYSTEM_UPDATE_INGESTION_INDICES_CLL_LIMIT:0}
   policyFields:
     enabled: ${BOOTSTRAP_SYSTEM_UPDATE_POLICY_FIELDS_ENABLED:true}
     batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_POLICY_FIELDS_BATCH_SIZE:5000}
@@ -405,6 +424,11 @@ systemUpdate:
     batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_EDGE_STATUS_BATCH_SIZE:1000}
     delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_EDGE_STATUS_DELAY_MS:5000}
     limit: ${BOOTSTRAP_SYSTEM_UPDATE_EDGE_STATUS_LIMIT:0}
+  propertyDefinitions:
+    enabled: ${BOOTSTRAP_SYSTEM_UPDATE_PROPERTY_DEFINITIONS_ENABLED:true}
+    batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_PROPERTY_DEFINITIONS_BATCH_SIZE:500}
+    delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_PROPERTY_DEFINITIONS_DELAY_MS:1000}
+    limit: ${BOOTSTRAP_SYSTEM_UPDATE_PROPERTY_DEFINITIONS_CLL_LIMIT:0}
       
 structuredProperties:
   enabled: ${ENABLE_STRUCTURED_PROPERTIES_HOOK:true} # applies structured properties mappings
@@ -435,8 +459,10 @@ featureFlags:
   schemaFieldEntityFetchEnabled: ${SCHEMA_FIELD_ENTITY_FETCH_ENABLED:true} # Enables fetching for schema field entities from the database when we hydrate them on schema fields
   businessAttributeEntityEnabled: ${BUSINESS_ATTRIBUTE_ENTITY_ENABLED:false} # Enables business attribute entity which can be associated with field of dataset
   dataContractsEnabled: ${DATA_CONTRACTS_ENABLED:true} # Enables the Data Contracts feature (Tab) in the UI
+  alternateMCPValidation: ${ALTERNATE_MCP_VALIDATION:false} # Enables alternate MCP validation flow
   showSeparateSiblings: ${SHOW_SEPARATE_SIBLINGS:false} # If turned on, all siblings will be separated with no way to get to a "combined" sibling view
   editableDatasetNameEnabled: ${EDITABLE_DATASET_NAME_ENABLED:false} # Enables the ability to edit the dataset name in the UI
+  showManageStructuredProperties: ${SHOW_MANAGE_STRUCTURED_PROPERTIES:true} # If turned on, show the manage structured properties button on the govern dropdown
 
 entityChangeEvents:
   enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true}
@@ -451,10 +477,19 @@ entityClient:
   java:
     get:
       batchSize: ${ENTITY_CLIENT_JAVA_GET_BATCH_SIZE:375} # matches EbeanAspectDao batch size
+    ingest:
+      batchSize: ${ENTITY_CLIENT_JAVA_INGEST_BATCH_SIZE:375}
   restli:
     get:
       batchSize: ${ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE:100} # limited to prevent exceeding restli URI size limit
       batchConcurrency: ${ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY:2} # parallel threads
+      batchQueueSize: ${ENTITY_CLIENT_RESTLI_GET_BATCH_QUEUE_SIZE:500}
+      batchThreadKeepAlive: ${ENTITY_CLIENT_RESTLI_GET_BATCH_THREAD_KEEP_ALIVE:60}
+    ingest:
+      batchSize: ${ENTITY_CLIENT_RESTLI_INGEST_BATCH_SIZE:50} # limited to prevent exceeding restli timeouts
+      batchConcurrency: ${ENTITY_CLIENT_RESTLI_INGEST_BATCH_CONCURRENCY:2} # parallel threads
+      batchQueueSize: ${ENTITY_CLIENT_RESTLI_INGEST_BATCH_QUEUE_SIZE:500}
+      batchThreadKeepAlive: ${ENTITY_CLIENT_RESTLI_INGEST_BATCH_THREAD_KEEP_ALIVE:60}
 
 usageClient:
   retryInterval: ${USAGE_CLIENT_RETRY_INTERVAL:2}
@@ -488,12 +523,12 @@ cache:
       entityAspectTTLSeconds:
         # cache user aspects for 20s
         corpuser:
-          corpUserKey: 20
+          corpUserKey: 300 # 5 min
           corpUserInfo: 20
           corpUserEditableInfo: 20
-          corpUserStatus: 20
+          corpUserStatus: 300 # 5 min
           globalTags: 20
-          status: 20
+          status: 300 # 5 min
           corpUserCredentials: 20
           corpUserSettings: 20
           roleMembership: 20
@@ -527,7 +562,7 @@ springdoc.api-docs.groups.enabled: true
 
 forms:
   hook:
-    enabled: { $FORMS_HOOK_ENABLED:true }
+    enabled: ${FORMS_HOOK_ENABLED:true}
     consumerGroupSuffix: ${FORMS_HOOK_CONSUMER_GROUP_SUFFIX:}
 
 businessAttribute:
@@ -539,6 +574,8 @@ businessAttribute:
 metadataChangeProposal:
   validation:
     ignoreUnknown: ${MCP_VALIDATION_IGNORE_UNKNOWN:true}
+    extensions:
+      enabled: ${MCP_VALIDATION_EXTENSIONS_ENABLED:false}
   sideEffects:
     schemaField:
       enabled: ${MCP_SIDE_EFFECTS_SCHEMA_FIELD_ENABLED:false}
diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml
index b1612f95f9219..a81cf39ce386f 100644
--- a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml
+++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml
@@ -13,24 +13,33 @@ bootstrap:
       mcps_location: "bootstrap_mcps/root-user.yaml"
 
     - name: data-platforms
-      version: v1
+      version: v2
+      blocking: true
+      async: false
       mcps_location: "bootstrap_mcps/data-platforms.yaml"
 
     - name: data-types
       version: v1
+      blocking: true
+      async: false
       mcps_location: "bootstrap_mcps/data-types.yaml"
 
     - name: ownership-types
       version: v1
+      blocking: true
+      async: false
       mcps_location: "bootstrap_mcps/ownership-types.yaml"
 
     - name: roles
       version: v1
+      blocking: true
+      async: false
       mcps_location: "bootstrap_mcps/roles.yaml"
 
     # Ingestion Recipes
     - name: ingestion-datahub-gc
-      version: v1
-      optional: true
+      version: v5
+      optional: false
       mcps_location: "bootstrap_mcps/ingestion-datahub-gc.yaml"
-      values_env: "DATAHUB_GC_BOOTSTRAP_VALUES"
\ No newline at end of file
+      values_env: "DATAHUB_GC_BOOTSTRAP_VALUES"
+      revision_env: "DATAHUB_GC_BOOTSTRAP_REVISION"
diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml
index 24d5da22805cb..2230d552ed4c0 100644
--- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml
+++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml
@@ -119,6 +119,16 @@
     displayName: Hive
     type: FILE_SYSTEM
     logoUrl: "/assets/platforms/hivelogo.png"
+- entityUrn: urn:li:dataPlatform:hudi
+  entityType: dataPlatform
+  aspectName: dataPlatformInfo
+  changeType: UPSERT
+  aspect:
+    datasetNameDelimiter: "."
+    name: hudi
+    displayName: Hudi
+    type: FILE_SYSTEM
+    logoUrl: "/assets/platforms/hudilogo.png"
 - entityUrn: urn:li:dataPlatform:iceberg
   entityType: dataPlatform
   aspectName: dataPlatformInfo
@@ -707,3 +717,34 @@
     displayName: SAP Analytics Cloud
     type: OTHERS
     logoUrl: "/assets/platforms/saclogo.svg"
+- entityUrn: urn:li:dataPlatform:dremio
+  entityType: dataPlatform
+  aspectName: dataPlatformInfo
+  changeType: UPSERT
+  aspect:
+    datasetNameDelimiter: "."
+    name: dremio
+    displayName: Dremio
+    type: QUERY_ENGINE
+    logoUrl: "/assets/platforms/dremiologo.png"
+- entityUrn: urn:li:dataPlatform:cassandra
+  entityType: dataPlatform
+  aspectName: dataPlatformInfo
+  changeType: UPSERT
+  aspect:
+    datasetNameDelimiter: "."
+    name: cassandra
+    displayName: Cassandra
+    type: KEY_VALUE_STORE
+    logoUrl: "/assets/platforms/cassandralogo.png"
+- entityUrn: urn:li:dataPlatform:neo4j
+  entityType: dataPlatform
+  aspectName: dataPlatformInfo
+  changeType: UPSERT
+  aspect:
+    datasetNameDelimiter: "."
+    name: neo4j
+    displayName: Neo4j
+    type: OTHERS
+    logoUrl: "/assets/platforms/neo4j.png"
+
diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml
index e70ab1162a381..c0c5be85b16b1 100644
--- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml
+++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml
@@ -12,13 +12,14 @@
       timezone: '{{schedule.timezone}}{{^schedule.timezone}}UTC{{/schedule.timezone}}'
       interval: '{{schedule.interval}}{{^schedule.interval}}0 1 * * *{{/schedule.interval}}'
     config:
-      version: 0.14.1.1rc5
+      version: '{{&ingestion.version}}{{^ingestion.version}}0.14.1.7rc2{{/ingestion.version}}'
       recipe:
         source:
           type: 'datahub-gc'
           config:
             cleanup_expired_tokens: {{cleanup_expired_tokens}}{{^cleanup_expired_tokens}}false{{/cleanup_expired_tokens}}
             truncate_indices: {{truncate_indices}}{{^truncate_indices}}true{{/truncate_indices}}
+            truncate_index_older_than_days: {{truncate_indices_retention_days}}{{^truncate_indices_retention_days}}30{{/truncate_indices_retention_days}}
             dataprocess_cleanup:
               retention_days: {{dataprocess_cleanup.retention_days}}{{^dataprocess_cleanup.retention_days}}10{{/dataprocess_cleanup.retention_days}}
               delete_empty_data_jobs: {{dataprocess_cleanup.delete_empty_data_jobs}}{{^dataprocess_cleanup.delete_empty_data_jobs}}true{{/dataprocess_cleanup.delete_empty_data_jobs}}
@@ -27,6 +28,12 @@
               keep_last_n: {{dataprocess_cleanup.keep_last_n}}{{^dataprocess_cleanup.keep_last_n}}5{{/dataprocess_cleanup.keep_last_n}}
             soft_deleted_entities_cleanup:
               retention_days: {{soft_deleted_entities_cleanup.retention_days}}{{^soft_deleted_entities_cleanup.retention_days}}10{{/soft_deleted_entities_cleanup.retention_days}}
+            execution_request_cleanup:
+              keep_history_min_count: {{execution_request_cleanup.keep_history_min_count}}{{^execution_request_cleanup.keep_history_min_count}}10{{/execution_request_cleanup.keep_history_min_count}}
+              keep_history_max_count: {{execution_request_cleanup.keep_history_max_count}}{{^execution_request_cleanup.keep_history_max_count}}1000{{/execution_request_cleanup.keep_history_max_count}}
+              keep_history_max_days: {{execution_request_cleanup.keep_history_max_days}}{{^execution_request_cleanup.keep_history_max_days}}30{{/execution_request_cleanup.keep_history_max_days}}
+              batch_read_size: {{execution_request_cleanup.batch_read_size}}{{^execution_request_cleanup.batch_read_size}}100{{/execution_request_cleanup.batch_read_size}}
+              enabled: {{execution_request_cleanup.enabled}}{{^execution_request_cleanup.enabled}}false{{/execution_request_cleanup.enabled}}
       extraArgs: {}
       debugMode: false
       executorId: default
diff --git a/metadata-service/configuration/src/main/resources/search_config.yaml b/metadata-service/configuration/src/main/resources/search_config.yaml
index e93f8af8b1d6c..47494c8cb1ca4 100644
--- a/metadata-service/configuration/src/main/resources/search_config.yaml
+++ b/metadata-service/configuration/src/main/resources/search_config.yaml
@@ -65,9 +65,9 @@ queryConfigurations:
       boost_mode: replace
 
   # Criteria for exact-match only
-  # Contains quotes, is a single term with `_`, `.`, or `-` (normally consider for tokenization) then use exact match query
+  # Contains quotes then use exact match query
   - queryRegex: >-
-      ^["'].+["']$|^[a-zA-Z0-9]\S+[_.-]\S+[a-zA-Z0-9]$
+      ^["'].+["']$
     simpleQuery: false
     prefixMatchQuery: true
     exactMatchQuery: true
diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle
index 5e52efd245b7f..501c46d64d6f9 100644
--- a/metadata-service/factories/build.gradle
+++ b/metadata-service/factories/build.gradle
@@ -1,6 +1,7 @@
 plugins {
   id 'java-library'
 }
+apply from: '../../gradle/coverage/java-coverage.gradle'
 
 dependencies {
   api project(':metadata-io')
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/CacheConfig.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/CacheConfig.java
index 383716a80cc60..545f8e087838d 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/CacheConfig.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/CacheConfig.java
@@ -48,7 +48,7 @@ private Caffeine<Object, Object> caffeineCacheBuilder() {
     return Caffeine.newBuilder()
         .initialCapacity(100)
         .maximumSize(cacheMaxSize)
-        .expireAfterAccess(cacheTtlSeconds, TimeUnit.SECONDS)
+        .expireAfterWrite(cacheTtlSeconds, TimeUnit.SECONDS)
         .recordStats();
   }
 
@@ -86,10 +86,7 @@ public CacheManager hazelcastCacheManager(
   @Bean
   @ConditionalOnProperty(name = "searchService.cacheImplementation", havingValue = "hazelcast")
   public MapConfig defaultMapConfig() {
-    // TODO: This setting is equivalent to expireAfterAccess, refreshes timer after a get, put,
-    // containsKey etc.
-    //       is this behavior what we actually desire? Should we change it now?
-    MapConfig mapConfig = new MapConfig().setMaxIdleSeconds(cacheTtlSeconds);
+    MapConfig mapConfig = new MapConfig().setTimeToLiveSeconds(cacheTtlSeconds);
 
     EvictionConfig evictionConfig =
         new EvictionConfig()
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java
index b8c4796dd4312..3e2823591e168 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java
@@ -16,6 +16,7 @@
 import io.datahubproject.metadata.context.OperationContextConfig;
 import io.datahubproject.metadata.context.RetrieverContext;
 import io.datahubproject.metadata.context.ServicesRegistryContext;
+import io.datahubproject.metadata.context.ValidationContext;
 import io.datahubproject.metadata.services.RestrictedService;
 import javax.annotation.Nonnull;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -43,7 +44,9 @@ protected OperationContext javaSystemOperationContext(
       @Nonnull final GraphService graphService,
       @Nonnull final SearchService searchService,
       @Qualifier("baseElasticSearchComponents")
-          BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components) {
+          BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components,
+      @Nonnull final ConfigurationProvider configurationProvider,
+      @Qualifier("systemEntityClient") @Nonnull final SystemEntityClient systemEntityClient) {
 
     EntityServiceAspectRetriever entityServiceAspectRetriever =
         EntityServiceAspectRetriever.builder()
@@ -51,6 +54,9 @@ protected OperationContext javaSystemOperationContext(
             .entityService(entityService)
             .build();
 
+    EntityClientAspectRetriever entityClientAspectRetriever =
+        EntityClientAspectRetriever.builder().entityClient(systemEntityClient).build();
+
     SystemGraphRetriever systemGraphRetriever =
         SystemGraphRetriever.builder().graphService(graphService).build();
 
@@ -66,10 +72,16 @@ protected OperationContext javaSystemOperationContext(
             components.getIndexConvention(),
             RetrieverContext.builder()
                 .aspectRetriever(entityServiceAspectRetriever)
+                .cachingAspectRetriever(entityClientAspectRetriever)
                 .graphRetriever(systemGraphRetriever)
                 .searchRetriever(searchServiceSearchRetriever)
+                .build(),
+            ValidationContext.builder()
+                .alternateValidation(
+                    configurationProvider.getFeatureFlags().isAlternateMCPValidation())
                 .build());
 
+    entityClientAspectRetriever.setSystemOperationContext(systemOperationContext);
     entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext);
     systemGraphRetriever.setSystemOperationContext(systemOperationContext);
     searchServiceSearchRetriever.setSystemOperationContext(systemOperationContext);
@@ -95,9 +107,10 @@ protected OperationContext restliSystemOperationContext(
       @Nonnull final GraphService graphService,
       @Nonnull final SearchService searchService,
       @Qualifier("baseElasticSearchComponents")
-          BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components) {
+          BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components,
+      @Nonnull final ConfigurationProvider configurationProvider) {
 
-    EntityClientAspectRetriever entityServiceAspectRetriever =
+    EntityClientAspectRetriever entityClientAspectRetriever =
         EntityClientAspectRetriever.builder().entityClient(systemEntityClient).build();
 
     SystemGraphRetriever systemGraphRetriever =
@@ -114,12 +127,16 @@ protected OperationContext restliSystemOperationContext(
             ServicesRegistryContext.builder().restrictedService(restrictedService).build(),
             components.getIndexConvention(),
             RetrieverContext.builder()
-                .aspectRetriever(entityServiceAspectRetriever)
+                .cachingAspectRetriever(entityClientAspectRetriever)
                 .graphRetriever(systemGraphRetriever)
                 .searchRetriever(searchServiceSearchRetriever)
+                .build(),
+            ValidationContext.builder()
+                .alternateValidation(
+                    configurationProvider.getFeatureFlags().isAlternateMCPValidation())
                 .build());
 
-    entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext);
+    entityClientAspectRetriever.setSystemOperationContext(systemOperationContext);
     systemGraphRetriever.setSystemOperationContext(systemOperationContext);
     searchServiceSearchRetriever.setSystemOperationContext(systemOperationContext);
 
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java
index c6ed9c625d6b5..4b87914012f40 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java
@@ -1,8 +1,11 @@
 package com.linkedin.gms.factory.entityclient;
 
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.gms.factory.config.ConfigurationProvider;
 import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig;
+import com.linkedin.parseq.retry.backoff.ExponentialBackoff;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -14,4 +17,30 @@ public EntityClientCacheConfig entityClientCacheConfig(
       @Qualifier("configurationProvider") final ConfigurationProvider configurationProvider) {
     return configurationProvider.getCache().getClient().getEntityClient();
   }
+
+  @Bean
+  public EntityClientConfig entityClientConfig(
+      final @Value("${entityClient.retryInterval:2}") int retryInterval,
+      final @Value("${entityClient.numRetries:3}") int numRetries,
+      final @Value("${entityClient.restli.get.batchSize}") int batchGetV2Size,
+      final @Value("${entityClient.restli.get.batchConcurrency}") int batchGetV2Concurrency,
+      final @Value("${entityClient.restli.get.batchQueueSize}") int batchGetV2QueueSize,
+      final @Value("${entityClient.restli.get.batchThreadKeepAlive}") int batchGetV2KeepAlive,
+      final @Value("${entityClient.restli.ingest.batchSize}") int batchIngestSize,
+      final @Value("${entityClient.restli.ingest.batchConcurrency}") int batchIngestConcurrency,
+      final @Value("${entityClient.restli.ingest.batchQueueSize}") int batchIngestQueueSize,
+      final @Value("${entityClient.restli.ingest.batchThreadKeepAlive}") int batchIngestKeepAlive) {
+    return EntityClientConfig.builder()
+        .backoffPolicy(new ExponentialBackoff(retryInterval))
+        .retryCount(numRetries)
+        .batchGetV2Size(batchGetV2Size)
+        .batchGetV2Concurrency(batchGetV2Concurrency)
+        .batchGetV2QueueSize(batchGetV2QueueSize)
+        .batchGetV2KeepAlive(batchGetV2KeepAlive)
+        .batchIngestSize(batchIngestSize)
+        .batchIngestConcurrency(batchIngestConcurrency)
+        .batchIngestQueueSize(batchIngestQueueSize)
+        .batchIngestKeepAlive(batchIngestKeepAlive)
+        .build();
+  }
 }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java
index fc35e6d045d0c..e783b4e1963d0 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java
@@ -1,6 +1,7 @@
 package com.linkedin.gms.factory.entityclient;
 
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.metadata.client.JavaEntityClient;
 import com.linkedin.metadata.client.SystemJavaEntityClient;
@@ -16,7 +17,6 @@
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import javax.inject.Singleton;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -39,7 +39,7 @@ public EntityClient entityClient(
       final @Qualifier("relationshipSearchService") LineageSearchService _lineageSearchService,
       final @Qualifier("kafkaEventProducer") EventProducer _eventProducer,
       final RollbackService rollbackService,
-      final @Value("${entityClient.restli.get.batchSize:375}") int batchGetV2Size) {
+      final EntityClientConfig entityClientConfig) {
     return new JavaEntityClient(
         _entityService,
         _deleteEntityService,
@@ -50,7 +50,7 @@ public EntityClient entityClient(
         _timeseriesAspectService,
         rollbackService,
         _eventProducer,
-        batchGetV2Size);
+        entityClientConfig);
   }
 
   @Bean("systemEntityClient")
@@ -67,7 +67,7 @@ public SystemEntityClient systemEntityClient(
       final @Qualifier("kafkaEventProducer") EventProducer _eventProducer,
       final RollbackService rollbackService,
       final EntityClientCacheConfig entityClientCacheConfig,
-      final @Value("${entityClient.restli.get.batchSize:375}") int batchGetV2Size) {
+      final EntityClientConfig entityClientConfig) {
     return new SystemJavaEntityClient(
         _entityService,
         _deleteEntityService,
@@ -79,6 +79,6 @@ public SystemEntityClient systemEntityClient(
         rollbackService,
         _eventProducer,
         entityClientCacheConfig,
-        batchGetV2Size);
+        entityClientConfig);
   }
 }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java
index 9e7255bf43a34..727dc9f6abc5b 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java
@@ -1,12 +1,12 @@
 package com.linkedin.gms.factory.entityclient;
 
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.entity.client.RestliEntityClient;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.entity.client.SystemRestliEntityClient;
 import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig;
 import com.linkedin.metadata.restli.DefaultRestliClientFactory;
-import com.linkedin.parseq.retry.backoff.ExponentialBackoff;
 import com.linkedin.restli.client.Client;
 import java.net.URI;
 import javax.inject.Singleton;
@@ -28,10 +28,7 @@ public EntityClient entityClient(
       @Value("${datahub.gms.useSSL}") boolean gmsUseSSL,
       @Value("${datahub.gms.uri}") String gmsUri,
       @Value("${datahub.gms.sslContext.protocol}") String gmsSslProtocol,
-      @Value("${entityClient.retryInterval:2}") int retryInterval,
-      @Value("${entityClient.numRetries:3}") int numRetries,
-      final @Value("${entityClient.restli.get.batchSize}") int batchGetV2Size,
-      final @Value("${entityClient.restli.get.batchConcurrency}") int batchGetV2Concurrency) {
+      final EntityClientConfig entityClientConfig) {
     final Client restClient;
     if (gmsUri != null) {
       restClient = DefaultRestliClientFactory.getRestLiClient(URI.create(gmsUri), gmsSslProtocol);
@@ -39,12 +36,7 @@ public EntityClient entityClient(
       restClient =
           DefaultRestliClientFactory.getRestLiClient(gmsHost, gmsPort, gmsUseSSL, gmsSslProtocol);
     }
-    return new RestliEntityClient(
-        restClient,
-        new ExponentialBackoff(retryInterval),
-        numRetries,
-        batchGetV2Size,
-        batchGetV2Concurrency);
+    return new RestliEntityClient(restClient, entityClientConfig);
   }
 
   @Bean("systemEntityClient")
@@ -55,11 +47,8 @@ public SystemEntityClient systemEntityClient(
       @Value("${datahub.gms.useSSL}") boolean gmsUseSSL,
       @Value("${datahub.gms.uri}") String gmsUri,
       @Value("${datahub.gms.sslContext.protocol}") String gmsSslProtocol,
-      @Value("${entityClient.retryInterval:2}") int retryInterval,
-      @Value("${entityClient.numRetries:3}") int numRetries,
       final EntityClientCacheConfig entityClientCacheConfig,
-      final @Value("${entityClient.restli.get.batchSize}") int batchGetV2Size,
-      final @Value("${entityClient.restli.get.batchConcurrency}") int batchGetV2Concurrency) {
+      final EntityClientConfig entityClientConfig) {
 
     final Client restClient;
     if (gmsUri != null) {
@@ -68,12 +57,6 @@ public SystemEntityClient systemEntityClient(
       restClient =
           DefaultRestliClientFactory.getRestLiClient(gmsHost, gmsPort, gmsUseSSL, gmsSslProtocol);
     }
-    return new SystemRestliEntityClient(
-        restClient,
-        new ExponentialBackoff(retryInterval),
-        numRetries,
-        entityClientCacheConfig,
-        batchGetV2Size,
-        batchGetV2Concurrency);
+    return new SystemRestliEntityClient(restClient, entityClientConfig, entityClientCacheConfig);
   }
 }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java
index 750af8ec488df..a1ee4df360b7e 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java
@@ -1,10 +1,18 @@
 package com.linkedin.gms.factory.kafka;
 
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.DEFAULT_EVENT_CONSUMER_NAME;
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.MCL_EVENT_CONSUMER_NAME;
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.MCP_EVENT_CONSUMER_NAME;
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.PE_EVENT_CONSUMER_NAME;
+
 import com.linkedin.gms.factory.config.ConfigurationProvider;
+import com.linkedin.metadata.config.kafka.ConsumerConfiguration;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import java.time.Duration;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Map;
+import javax.annotation.Nullable;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -23,7 +31,6 @@
 @Slf4j
 @Configuration
 public class KafkaEventConsumerFactory {
-
   private int kafkaEventConsumerConcurrency;
 
   @Bean(name = "kafkaConsumerFactory")
@@ -87,15 +94,82 @@ private static Map<String, Object> buildCustomizedProperties(
     return customizedProperties;
   }
 
-  @Bean(name = "kafkaEventConsumer")
+  @Bean(name = PE_EVENT_CONSUMER_NAME)
+  protected KafkaListenerContainerFactory<?> platformEventConsumer(
+      @Qualifier("kafkaConsumerFactory")
+          DefaultKafkaConsumerFactory<String, GenericRecord> kafkaConsumerFactory,
+      @Qualifier("configurationProvider") ConfigurationProvider configurationProvider) {
+
+    return buildDefaultKafkaListenerContainerFactory(
+        PE_EVENT_CONSUMER_NAME,
+        kafkaConsumerFactory,
+        configurationProvider.getKafka().getConsumer().isStopOnDeserializationError(),
+        configurationProvider.getKafka().getConsumer().getPe());
+  }
+
+  @Bean(name = MCP_EVENT_CONSUMER_NAME)
+  protected KafkaListenerContainerFactory<?> mcpEventConsumer(
+      @Qualifier("kafkaConsumerFactory")
+          DefaultKafkaConsumerFactory<String, GenericRecord> kafkaConsumerFactory,
+      @Qualifier("configurationProvider") ConfigurationProvider configurationProvider) {
+
+    return buildDefaultKafkaListenerContainerFactory(
+        MCP_EVENT_CONSUMER_NAME,
+        kafkaConsumerFactory,
+        configurationProvider.getKafka().getConsumer().isStopOnDeserializationError(),
+        configurationProvider.getKafka().getConsumer().getMcp());
+  }
+
+  @Bean(name = MCL_EVENT_CONSUMER_NAME)
+  protected KafkaListenerContainerFactory<?> mclEventConsumer(
+      @Qualifier("kafkaConsumerFactory")
+          DefaultKafkaConsumerFactory<String, GenericRecord> kafkaConsumerFactory,
+      @Qualifier("configurationProvider") ConfigurationProvider configurationProvider) {
+
+    return buildDefaultKafkaListenerContainerFactory(
+        MCL_EVENT_CONSUMER_NAME,
+        kafkaConsumerFactory,
+        configurationProvider.getKafka().getConsumer().isStopOnDeserializationError(),
+        configurationProvider.getKafka().getConsumer().getMcl());
+  }
+
+  @Bean(name = DEFAULT_EVENT_CONSUMER_NAME)
   protected KafkaListenerContainerFactory<?> kafkaEventConsumer(
       @Qualifier("kafkaConsumerFactory")
           DefaultKafkaConsumerFactory<String, GenericRecord> kafkaConsumerFactory,
       @Qualifier("configurationProvider") ConfigurationProvider configurationProvider) {
 
+    return buildDefaultKafkaListenerContainerFactory(
+        DEFAULT_EVENT_CONSUMER_NAME,
+        kafkaConsumerFactory,
+        configurationProvider.getKafka().getConsumer().isStopOnDeserializationError(),
+        null);
+  }
+
+  private KafkaListenerContainerFactory<?> buildDefaultKafkaListenerContainerFactory(
+      String consumerFactoryName,
+      DefaultKafkaConsumerFactory<String, GenericRecord> kafkaConsumerFactory,
+      boolean isStopOnDeserializationError,
+      @Nullable ConsumerConfiguration.ConsumerOptions consumerOptions) {
+
+    final DefaultKafkaConsumerFactory<String, GenericRecord> factoryWithOverrides;
+    if (consumerOptions != null) {
+      // Copy the base config
+      Map<String, Object> props = new HashMap<>(kafkaConsumerFactory.getConfigurationProperties());
+      // Override just the auto.offset.reset
+      props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, consumerOptions.getAutoOffsetReset());
+      factoryWithOverrides =
+          new DefaultKafkaConsumerFactory<>(
+              props,
+              kafkaConsumerFactory.getKeyDeserializer(),
+              kafkaConsumerFactory.getValueDeserializer());
+    } else {
+      factoryWithOverrides = kafkaConsumerFactory;
+    }
+
     ConcurrentKafkaListenerContainerFactory<String, GenericRecord> factory =
         new ConcurrentKafkaListenerContainerFactory<>();
-    factory.setConsumerFactory(kafkaConsumerFactory);
+    factory.setConsumerFactory(factoryWithOverrides);
     factory.setContainerCustomizer(new ThreadPoolContainerCustomizer());
     factory.setConcurrency(kafkaEventConsumerConcurrency);
 
@@ -103,7 +177,7 @@ protected KafkaListenerContainerFactory<?> kafkaEventConsumer(
      use DefaultErrorHandler (does back-off retry and then logs) rather than stopping the container. Stopping the container
      prevents lost messages until the error can be examined, disabling this will allow progress, but may lose data
     */
-    if (configurationProvider.getKafka().getConsumer().isStopOnDeserializationError()) {
+    if (isStopOnDeserializationError) {
       CommonDelegatingErrorHandler delegatingErrorHandler =
           new CommonDelegatingErrorHandler(new DefaultErrorHandler());
       delegatingErrorHandler.addDelegate(
@@ -111,9 +185,9 @@ use DefaultErrorHandler (does back-off retry and then logs) rather than stopping
       factory.setCommonErrorHandler(delegatingErrorHandler);
     }
     log.info(
-        String.format(
-            "Event-based KafkaListenerContainerFactory built successfully. Consumer concurrency = %s",
-            kafkaEventConsumerConcurrency));
+        "Event-based {} KafkaListenerContainerFactory built successfully. Consumer concurrency = {}",
+        consumerFactoryName,
+        kafkaEventConsumerConcurrency);
 
     return factory;
   }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java
index 4a2095685abe1..2349dbd169f1d 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java
@@ -1,14 +1,24 @@
 package com.linkedin.gms.factory.plugins;
 
+import static com.linkedin.metadata.Constants.EDITABLE_SCHEMA_METADATA_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_ENTITY_NAME;
+import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME;
 import static com.linkedin.metadata.Constants.SCHEMA_METADATA_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME;
 
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.aspect.hooks.IgnoreUnknownMutator;
 import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
 import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect;
 import com.linkedin.metadata.aspect.plugins.hooks.MutationHook;
+import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
+import com.linkedin.metadata.aspect.validation.ExecutionRequestResultValidator;
+import com.linkedin.metadata.aspect.validation.FieldPathValidator;
 import com.linkedin.metadata.dataproducts.sideeffects.DataProductUnsetSideEffect;
 import com.linkedin.metadata.schemafields.sideeffects.SchemaFieldSideEffect;
+import com.linkedin.metadata.structuredproperties.validation.HidePropertyValidator;
+import com.linkedin.metadata.structuredproperties.validation.ShowPropertyAsBadgeValidator;
 import com.linkedin.metadata.timeline.eventgenerator.EntityChangeEventGeneratorRegistry;
 import com.linkedin.metadata.timeline.eventgenerator.SchemaMetadataChangeEventGenerator;
 import java.util.List;
@@ -21,6 +31,7 @@
 @Configuration
 @Slf4j
 public class SpringStandardPluginConfiguration {
+  private static final String ALL = "*";
 
   @Value("${metadataChangeProposal.validation.ignoreUnknown}")
   private boolean ignoreUnknownEnabled;
@@ -38,7 +49,7 @@ public MutationHook ignoreUnknownMutator() {
             AspectPluginConfig.builder()
                 .className(IgnoreUnknownMutator.class.getName())
                 .enabled(ignoreUnknownEnabled && !extensionsEnabled)
-                .supportedOperations(List.of("CREATE", "CREATE_ENTITY", "UPSERT"))
+                .supportedOperations(List.of("*"))
                 .supportedEntityAspectNames(
                     List.of(
                         AspectPluginConfig.EntityAspectName.builder()
@@ -91,8 +102,7 @@ public MCPSideEffect dataProductUnsetSideEffect() {
         AspectPluginConfig.builder()
             .enabled(true)
             .className(DataProductUnsetSideEffect.class.getName())
-            .supportedOperations(
-                List.of("CREATE", "CREATE_ENTITY", "UPSERT", "RESTATE", "DELETE", "PATCH"))
+            .supportedOperations(List.of("CREATE", "CREATE_ENTITY", "UPSERT", "RESTATE"))
             .supportedEntityAspectNames(
                 List.of(
                     AspectPluginConfig.EntityAspectName.builder()
@@ -104,4 +114,79 @@ public MCPSideEffect dataProductUnsetSideEffect() {
     log.info("Initialized {}", SchemaFieldSideEffect.class.getName());
     return new DataProductUnsetSideEffect().setConfig(config);
   }
+
+  @Bean
+  public AspectPayloadValidator fieldPathValidator() {
+    return new FieldPathValidator()
+        .setConfig(
+            AspectPluginConfig.builder()
+                .className(FieldPathValidator.class.getName())
+                .enabled(true)
+                .supportedOperations(
+                    List.of("CREATE", "CREATE_ENTITY", "UPSERT", "UPDATE", "RESTATE"))
+                .supportedEntityAspectNames(
+                    List.of(
+                        AspectPluginConfig.EntityAspectName.builder()
+                            .entityName(ALL)
+                            .aspectName(SCHEMA_METADATA_ASPECT_NAME)
+                            .build(),
+                        AspectPluginConfig.EntityAspectName.builder()
+                            .entityName(ALL)
+                            .aspectName(EDITABLE_SCHEMA_METADATA_ASPECT_NAME)
+                            .build()))
+                .build());
+  }
+
+  @Bean
+  public AspectPayloadValidator dataHubExecutionRequestResultValidator() {
+    return new ExecutionRequestResultValidator()
+        .setConfig(
+            AspectPluginConfig.builder()
+                .className(ExecutionRequestResultValidator.class.getName())
+                .enabled(true)
+                .supportedOperations(List.of("UPSERT", "UPDATE"))
+                .supportedEntityAspectNames(
+                    List.of(
+                        AspectPluginConfig.EntityAspectName.builder()
+                            .entityName(EXECUTION_REQUEST_ENTITY_NAME)
+                            .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME)
+                            .build()))
+                .build());
+  }
+
+  @Bean
+  public AspectPayloadValidator hidePropertyValidator() {
+    return new HidePropertyValidator()
+        .setConfig(
+            AspectPluginConfig.builder()
+                .className(HidePropertyValidator.class.getName())
+                .enabled(true)
+                .supportedOperations(
+                    List.of("UPSERT", "UPDATE", "CREATE", "CREATE_ENTITY", "RESTATE", "PATCH"))
+                .supportedEntityAspectNames(
+                    List.of(
+                        AspectPluginConfig.EntityAspectName.builder()
+                            .entityName(STRUCTURED_PROPERTY_ENTITY_NAME)
+                            .aspectName(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME)
+                            .build()))
+                .build());
+  }
+
+  @Bean
+  public AspectPayloadValidator showPropertyAsAssetBadgeValidator() {
+    return new ShowPropertyAsBadgeValidator()
+        .setConfig(
+            AspectPluginConfig.builder()
+                .className(ShowPropertyAsBadgeValidator.class.getName())
+                .enabled(true)
+                .supportedOperations(
+                    List.of("UPSERT", "UPDATE", "CREATE", "CREATE_ENTITY", "RESTATE", "PATCH"))
+                .supportedEntityAspectNames(
+                    List.of(
+                        AspectPluginConfig.EntityAspectName.builder()
+                            .entityName(STRUCTURED_PROPERTY_ENTITY_NAME)
+                            .aspectName(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME)
+                            .build()))
+                .build());
+  }
 }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeseries/ElasticSearchTimeseriesAspectServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeseries/ElasticSearchTimeseriesAspectServiceFactory.java
index e26de0e730195..85c58533df85f 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeseries/ElasticSearchTimeseriesAspectServiceFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeseries/ElasticSearchTimeseriesAspectServiceFactory.java
@@ -1,5 +1,6 @@
 package com.linkedin.gms.factory.timeseries;
 
+import com.linkedin.gms.factory.config.ConfigurationProvider;
 import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory;
 import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory;
 import com.linkedin.metadata.models.registry.EntityRegistry;
@@ -27,13 +28,15 @@ public class ElasticSearchTimeseriesAspectServiceFactory {
   @Bean(name = "elasticSearchTimeseriesAspectService")
   @Nonnull
   protected ElasticSearchTimeseriesAspectService getInstance(
-      final QueryFilterRewriteChain queryFilterRewriteChain) {
+      final QueryFilterRewriteChain queryFilterRewriteChain,
+      final ConfigurationProvider configurationProvider) {
     return new ElasticSearchTimeseriesAspectService(
         components.getSearchClient(),
         new TimeseriesAspectIndexBuilders(
             components.getIndexBuilder(), entityRegistry, components.getIndexConvention()),
         components.getBulkProcessor(),
         components.getNumRetries(),
-        queryFilterRewriteChain);
+        queryFilterRewriteChain,
+        configurationProvider.getTimeseriesAspectService());
   }
 }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
index ffc739e905cd6..3f8d24b9b61a9 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
@@ -18,6 +18,7 @@
 import com.linkedin.metadata.boot.steps.RemoveClientIdAspectStep;
 import com.linkedin.metadata.boot.steps.RestoreColumnLineageIndices;
 import com.linkedin.metadata.boot.steps.RestoreDbtSiblingsIndices;
+import com.linkedin.metadata.boot.steps.RestoreFormInfoIndicesStep;
 import com.linkedin.metadata.boot.steps.RestoreGlossaryIndices;
 import com.linkedin.metadata.boot.steps.WaitForSystemUpdateStep;
 import com.linkedin.metadata.entity.AspectMigrationsDao;
@@ -110,6 +111,8 @@ protected BootstrapManager createInstance(
     final WaitForSystemUpdateStep waitForSystemUpdateStep =
         new WaitForSystemUpdateStep(_dataHubUpgradeKafkaListener, _configurationProvider);
     final IngestEntityTypesStep ingestEntityTypesStep = new IngestEntityTypesStep(_entityService);
+    final RestoreFormInfoIndicesStep restoreFormInfoIndicesStep =
+        new RestoreFormInfoIndicesStep(_entityService);
 
     final List<BootstrapStep> finalSteps =
         new ArrayList<>(
@@ -124,7 +127,8 @@ protected BootstrapManager createInstance(
                 restoreDbtSiblingsIndices,
                 indexDataPlatformsStep,
                 restoreColumnLineageIndices,
-                ingestEntityTypesStep));
+                ingestEntityTypesStep,
+                restoreFormInfoIndicesStep));
 
     return new BootstrapManager(finalSteps);
   }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java
index 22ce06a5984ea..c04dd25ccd4ac 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java
@@ -84,14 +84,14 @@ public void execute(@Nonnull OperationContext systemOperationContext) throws Exc
                   .aspectName(DATA_PLATFORM_INSTANCE_ASPECT_NAME)
                   .recordTemplate(dataPlatformInstance.get())
                   .auditStamp(aspectAuditStamp)
-                  .build(systemOperationContext.getAspectRetrieverOpt().get()));
+                  .build(systemOperationContext.getAspectRetriever()));
         }
       }
 
       _entityService.ingestAspects(
           systemOperationContext,
           AspectsBatchImpl.builder()
-              .retrieverContext(systemOperationContext.getRetrieverContext().get())
+              .retrieverContext(systemOperationContext.getRetrieverContext())
               .items(items)
               .build(),
           true,
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java
index eb6bfe17ac198..dac2879487469 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java
@@ -225,7 +225,7 @@ private void ingestPolicy(
                 new AuditStamp()
                     .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR))
                     .setTime(System.currentTimeMillis()),
-                systemOperationContext.getRetrieverContext().get())
+                systemOperationContext.getRetrieverContext())
             .build(),
         false);
   }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreFormInfoIndicesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreFormInfoIndicesStep.java
new file mode 100644
index 0000000000000..c5dbea5965a66
--- /dev/null
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreFormInfoIndicesStep.java
@@ -0,0 +1,129 @@
+package com.linkedin.metadata.boot.steps;
+
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.data.template.RecordTemplate;
+import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.form.FormInfo;
+import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.boot.UpgradeStep;
+import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.entity.ListResult;
+import com.linkedin.metadata.models.AspectSpec;
+import com.linkedin.metadata.query.ExtraInfo;
+import io.datahubproject.metadata.context.OperationContext;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class RestoreFormInfoIndicesStep extends UpgradeStep {
+  private static final String VERSION = "1";
+  private static final String UPGRADE_ID = "restore-form-info-indices";
+  private static final Integer BATCH_SIZE = 1000;
+
+  public RestoreFormInfoIndicesStep(@Nonnull final EntityService<?> entityService) {
+    super(entityService, VERSION, UPGRADE_ID);
+  }
+
+  @Override
+  public void upgrade(@Nonnull OperationContext systemOperationContext) throws Exception {
+    final AuditStamp auditStamp =
+        new AuditStamp()
+            .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR))
+            .setTime(System.currentTimeMillis());
+
+    final int totalFormCount = getAndRestoreFormInfoIndices(systemOperationContext, 0, auditStamp);
+    int formCount = BATCH_SIZE;
+    while (formCount < totalFormCount) {
+      getAndRestoreFormInfoIndices(systemOperationContext, formCount, auditStamp);
+      formCount += BATCH_SIZE;
+    }
+  }
+
+  @Nonnull
+  @Override
+  public ExecutionMode getExecutionMode() {
+    return ExecutionMode.ASYNC;
+  }
+
+  private int getAndRestoreFormInfoIndices(
+      @Nonnull OperationContext systemOperationContext, int start, AuditStamp auditStamp) {
+    final AspectSpec formInfoAspectSpec =
+        systemOperationContext
+            .getEntityRegistry()
+            .getEntitySpec(Constants.FORM_ENTITY_NAME)
+            .getAspectSpec(Constants.FORM_INFO_ASPECT_NAME);
+
+    final ListResult<RecordTemplate> latestAspects =
+        entityService.listLatestAspects(
+            systemOperationContext,
+            Constants.FORM_ENTITY_NAME,
+            Constants.FORM_INFO_ASPECT_NAME,
+            start,
+            BATCH_SIZE);
+
+    if (latestAspects.getTotalCount() == 0
+        || latestAspects.getValues() == null
+        || latestAspects.getMetadata() == null) {
+      log.debug("Found 0 formInfo aspects for forms. Skipping migration.");
+      return 0;
+    }
+
+    if (latestAspects.getValues().size() != latestAspects.getMetadata().getExtraInfos().size()) {
+      // Bad result -- we should log that we cannot migrate this batch of formInfos.
+      log.warn(
+          "Failed to match formInfo aspects with corresponding urns. Found mismatched length between aspects ({})"
+              + "and metadata ({}) for metadata {}",
+          latestAspects.getValues().size(),
+          latestAspects.getMetadata().getExtraInfos().size(),
+          latestAspects.getMetadata());
+      return latestAspects.getTotalCount();
+    }
+
+    List<Future<?>> futures = new LinkedList<>();
+    for (int i = 0; i < latestAspects.getValues().size(); i++) {
+      ExtraInfo info = latestAspects.getMetadata().getExtraInfos().get(i);
+      RecordTemplate formInfoRecord = latestAspects.getValues().get(i);
+      Urn urn = info.getUrn();
+      FormInfo formInfo = (FormInfo) formInfoRecord;
+      if (formInfo == null) {
+        log.warn("Received null formInfo for urn {}", urn);
+        continue;
+      }
+
+      futures.add(
+          entityService
+              .alwaysProduceMCLAsync(
+                  systemOperationContext,
+                  urn,
+                  Constants.FORM_ENTITY_NAME,
+                  Constants.FORM_INFO_ASPECT_NAME,
+                  formInfoAspectSpec,
+                  null,
+                  formInfo,
+                  null,
+                  null,
+                  auditStamp,
+                  ChangeType.RESTATE)
+              .getFirst());
+    }
+
+    futures.stream()
+        .filter(Objects::nonNull)
+        .forEach(
+            f -> {
+              try {
+                f.get();
+              } catch (InterruptedException | ExecutionException e) {
+                throw new RuntimeException(e);
+              }
+            });
+
+    return latestAspects.getTotalCount();
+  }
+}
diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreFormInfoIndicesStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreFormInfoIndicesStepTest.java
new file mode 100644
index 0000000000000..9705ecbfcb739
--- /dev/null
+++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreFormInfoIndicesStepTest.java
@@ -0,0 +1,242 @@
+package com.linkedin.metadata.boot.steps;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.entity.Aspect;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.entity.EnvelopedAspectMap;
+import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.form.FormInfo;
+import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.entity.ListResult;
+import com.linkedin.metadata.models.AspectSpec;
+import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import com.linkedin.metadata.query.ExtraInfo;
+import com.linkedin.metadata.query.ExtraInfoArray;
+import com.linkedin.metadata.query.ListResultMetadata;
+import com.linkedin.mxe.MetadataChangeProposal;
+import com.linkedin.util.Pair;
+import io.datahubproject.metadata.context.OperationContext;
+import jakarta.annotation.Nonnull;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Future;
+import org.mockito.Mockito;
+import org.testng.annotations.Test;
+
+public class RestoreFormInfoIndicesStepTest {
+
+  private static final String VERSION_1 = "1";
+  private static final String VERSION_2 = "2";
+  private static final String FORM_INFO_UPGRADE_URN =
+      String.format(
+          "urn:li:%s:%s", Constants.DATA_HUB_UPGRADE_ENTITY_NAME, "restore-form-info-indices");
+  private final Urn formUrn = UrnUtils.getUrn("urn:li:form:test");
+
+  @Test
+  public void testExecuteFirstTime() throws Exception {
+    final EntityService<?> mockService = Mockito.mock(EntityService.class);
+    final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class);
+    final OperationContext mockContext = mock(OperationContext.class);
+    when(mockContext.getEntityRegistry()).thenReturn(mockRegistry);
+
+    mockGetUpgradeStep(mockContext, false, VERSION_1, mockService);
+    mockGetFormInfo(mockContext, formUrn, mockService);
+
+    final AspectSpec aspectSpec = mockAspectSpecs(mockRegistry);
+
+    final RestoreFormInfoIndicesStep restoreIndicesStep =
+        new RestoreFormInfoIndicesStep(mockService);
+    restoreIndicesStep.execute(mockContext);
+
+    Mockito.verify(mockRegistry, Mockito.times(1)).getEntitySpec(Constants.FORM_ENTITY_NAME);
+    // creates upgradeRequest and upgradeResult aspects
+    Mockito.verify(mockService, Mockito.times(2))
+        .ingestProposal(
+            any(OperationContext.class),
+            any(MetadataChangeProposal.class),
+            any(AuditStamp.class),
+            Mockito.eq(false));
+    Mockito.verify(mockService, Mockito.times(1))
+        .alwaysProduceMCLAsync(
+            any(OperationContext.class),
+            Mockito.eq(formUrn),
+            Mockito.eq(Constants.FORM_ENTITY_NAME),
+            Mockito.eq(Constants.FORM_INFO_ASPECT_NAME),
+            Mockito.eq(aspectSpec),
+            Mockito.eq(null),
+            any(),
+            Mockito.eq(null),
+            Mockito.eq(null),
+            any(),
+            Mockito.eq(ChangeType.RESTATE));
+  }
+
+  @Test
+  public void testExecuteWithNewVersion() throws Exception {
+    final EntityService<?> mockService = Mockito.mock(EntityService.class);
+    final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class);
+    final OperationContext mockContext = mock(OperationContext.class);
+    when(mockContext.getEntityRegistry()).thenReturn(mockRegistry);
+
+    mockGetUpgradeStep(mockContext, true, VERSION_2, mockService);
+    mockGetFormInfo(mockContext, formUrn, mockService);
+
+    final AspectSpec aspectSpec = mockAspectSpecs(mockRegistry);
+
+    final RestoreFormInfoIndicesStep restoreIndicesStep =
+        new RestoreFormInfoIndicesStep(mockService);
+    restoreIndicesStep.execute(mockContext);
+
+    Mockito.verify(mockRegistry, Mockito.times(1)).getEntitySpec(Constants.FORM_ENTITY_NAME);
+    // creates upgradeRequest and upgradeResult aspects
+    Mockito.verify(mockService, Mockito.times(2))
+        .ingestProposal(
+            any(OperationContext.class),
+            any(MetadataChangeProposal.class),
+            any(AuditStamp.class),
+            Mockito.eq(false));
+    Mockito.verify(mockService, Mockito.times(1))
+        .alwaysProduceMCLAsync(
+            any(OperationContext.class),
+            Mockito.eq(formUrn),
+            Mockito.eq(Constants.FORM_ENTITY_NAME),
+            Mockito.eq(Constants.FORM_INFO_ASPECT_NAME),
+            Mockito.eq(aspectSpec),
+            Mockito.eq(null),
+            any(),
+            Mockito.eq(null),
+            Mockito.eq(null),
+            any(),
+            Mockito.eq(ChangeType.RESTATE));
+  }
+
+  @Test
+  public void testDoesNotExecuteWithSameVersion() throws Exception {
+    final EntityService<?> mockService = Mockito.mock(EntityService.class);
+    final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class);
+    final OperationContext mockContext = mock(OperationContext.class);
+    when(mockContext.getEntityRegistry()).thenReturn(mockRegistry);
+
+    mockGetUpgradeStep(mockContext, true, VERSION_1, mockService);
+    mockGetFormInfo(mockContext, formUrn, mockService);
+
+    final AspectSpec aspectSpec = mockAspectSpecs(mockRegistry);
+
+    final RestoreFormInfoIndicesStep restoreIndicesStep =
+        new RestoreFormInfoIndicesStep(mockService);
+    restoreIndicesStep.execute(mockContext);
+
+    Mockito.verify(mockRegistry, Mockito.times(0)).getEntitySpec(Constants.FORM_ENTITY_NAME);
+    // creates upgradeRequest and upgradeResult aspects
+    Mockito.verify(mockService, Mockito.times(0))
+        .ingestProposal(
+            any(OperationContext.class),
+            any(MetadataChangeProposal.class),
+            any(AuditStamp.class),
+            Mockito.eq(false));
+    Mockito.verify(mockService, Mockito.times(0))
+        .alwaysProduceMCLAsync(
+            any(OperationContext.class),
+            Mockito.eq(formUrn),
+            Mockito.eq(Constants.FORM_ENTITY_NAME),
+            Mockito.eq(Constants.FORM_INFO_ASPECT_NAME),
+            Mockito.eq(aspectSpec),
+            Mockito.eq(null),
+            any(),
+            Mockito.eq(null),
+            Mockito.eq(null),
+            any(),
+            Mockito.eq(ChangeType.RESTATE));
+  }
+
+  private void mockGetFormInfo(
+      @Nonnull OperationContext mockContext,
+      @Nonnull Urn formUrn,
+      @Nonnull EntityService<?> mockService) {
+    final List<ExtraInfo> extraInfos =
+        ImmutableList.of(
+            new ExtraInfo()
+                .setUrn(formUrn)
+                .setVersion(0L)
+                .setAudit(
+                    new AuditStamp()
+                        .setActor(UrnUtils.getUrn("urn:li:corpuser:test"))
+                        .setTime(0L)));
+
+    when(mockService.alwaysProduceMCLAsync(
+            any(OperationContext.class),
+            any(Urn.class),
+            Mockito.anyString(),
+            Mockito.anyString(),
+            any(AspectSpec.class),
+            Mockito.eq(null),
+            any(),
+            any(),
+            any(),
+            any(),
+            any(ChangeType.class)))
+        .thenReturn(Pair.of(Mockito.mock(Future.class), false));
+
+    when(mockService.listLatestAspects(
+            any(OperationContext.class),
+            Mockito.eq(Constants.FORM_ENTITY_NAME),
+            Mockito.eq(Constants.FORM_INFO_ASPECT_NAME),
+            Mockito.eq(0),
+            Mockito.eq(1000)))
+        .thenReturn(
+            new ListResult<>(
+                ImmutableList.of(new FormInfo()),
+                new ListResultMetadata().setExtraInfos(new ExtraInfoArray(extraInfos)),
+                1,
+                false,
+                1,
+                1,
+                1));
+  }
+
+  private AspectSpec mockAspectSpecs(@Nonnull EntityRegistry mockRegistry) {
+    final EntitySpec entitySpec = Mockito.mock(EntitySpec.class);
+    final AspectSpec aspectSpec = Mockito.mock(AspectSpec.class);
+    //  Mock for formInfo
+    when(mockRegistry.getEntitySpec(Constants.FORM_ENTITY_NAME)).thenReturn(entitySpec);
+    when(entitySpec.getAspectSpec(Constants.FORM_INFO_ASPECT_NAME)).thenReturn(aspectSpec);
+
+    return aspectSpec;
+  }
+
+  private void mockGetUpgradeStep(
+      @Nonnull OperationContext mockContext,
+      boolean shouldReturnResponse,
+      @Nonnull String version,
+      @Nonnull EntityService<?> mockService)
+      throws Exception {
+
+    final Urn upgradeEntityUrn = UrnUtils.getUrn(FORM_INFO_UPGRADE_URN);
+    final com.linkedin.upgrade.DataHubUpgradeRequest upgradeRequest =
+        new com.linkedin.upgrade.DataHubUpgradeRequest().setVersion(version);
+    final Map<String, EnvelopedAspect> upgradeRequestAspects = new HashMap<>();
+    upgradeRequestAspects.put(
+        Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME,
+        new EnvelopedAspect().setValue(new Aspect(upgradeRequest.data())));
+    final EntityResponse response =
+        new EntityResponse().setAspects(new EnvelopedAspectMap(upgradeRequestAspects));
+    when(mockService.getEntityV2(
+            mockContext,
+            Constants.DATA_HUB_UPGRADE_ENTITY_NAME,
+            upgradeEntityUrn,
+            Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)))
+        .thenReturn(shouldReturnResponse ? response : null);
+  }
+}
diff --git a/metadata-service/graphql-servlet-impl/build.gradle b/metadata-service/graphql-servlet-impl/build.gradle
index 5767698242118..6848696db6788 100644
--- a/metadata-service/graphql-servlet-impl/build.gradle
+++ b/metadata-service/graphql-servlet-impl/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(':datahub-graphql-core')
   implementation project(':metadata-auth:auth-api')
diff --git a/metadata-service/openapi-analytics-servlet/build.gradle b/metadata-service/openapi-analytics-servlet/build.gradle
index 5d1372a293775..45e0ee1758ebc 100644
--- a/metadata-service/openapi-analytics-servlet/build.gradle
+++ b/metadata-service/openapi-analytics-servlet/build.gradle
@@ -3,6 +3,8 @@ plugins {
     id 'org.hidetake.swagger.generator'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
 
     implementation project(':metadata-auth:auth-api')
diff --git a/metadata-service/openapi-entity-servlet/build.gradle b/metadata-service/openapi-entity-servlet/build.gradle
index 4c2d587a81fd7..a47e487f871c6 100644
--- a/metadata-service/openapi-entity-servlet/build.gradle
+++ b/metadata-service/openapi-entity-servlet/build.gradle
@@ -3,6 +3,8 @@ plugins {
   id 'org.hidetake.swagger.generator'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
 
   implementation project(':metadata-auth:auth-api')
diff --git a/metadata-service/openapi-servlet/build.gradle b/metadata-service/openapi-servlet/build.gradle
index e26b1ceea1a3c..77679790f25de 100644
--- a/metadata-service/openapi-servlet/build.gradle
+++ b/metadata-service/openapi-servlet/build.gradle
@@ -3,6 +3,8 @@ plugins {
   id 'org.hidetake.swagger.generator'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
 
   implementation project(':metadata-auth:auth-api')
diff --git a/metadata-service/openapi-servlet/models/build.gradle b/metadata-service/openapi-servlet/models/build.gradle
index e4100b2d094e0..d75e656e5ecd6 100644
--- a/metadata-service/openapi-servlet/models/build.gradle
+++ b/metadata-service/openapi-servlet/models/build.gradle
@@ -10,6 +10,7 @@ dependencies {
   implementation externalDependency.jacksonDataBind
   implementation externalDependency.httpClient
 
+  compileOnly externalDependency.swaggerAnnotations
   compileOnly externalDependency.lombok
 
   annotationProcessor externalDependency.lombok
diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityV2.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityV2.java
index 83c23f3552409..e281a39f764e5 100644
--- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityV2.java
+++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityV2.java
@@ -3,6 +3,7 @@
 import com.datahub.util.RecordUtils;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.mxe.SystemMetadata;
@@ -37,29 +38,42 @@ public static class GenericEntityV2Builder {
 
     public GenericEntityV2 build(
         ObjectMapper objectMapper, Map<String, Pair<RecordTemplate, SystemMetadata>> aspects) {
+      return build(objectMapper, aspects, false);
+    }
+
+    public GenericEntityV2 build(
+        ObjectMapper objectMapper,
+        Map<String, Pair<RecordTemplate, SystemMetadata>> aspects,
+        boolean isAsyncAlternateValidation) {
       Map<String, GenericAspectV2> jsonObjectMap =
           aspects.entrySet().stream()
               .map(
                   e -> {
                     try {
-                      Map<String, Object> valueMap =
+                      Map<String, JsonNode> valueMap =
                           Map.of(
                               "value",
                               objectMapper.readTree(
                                   RecordUtils.toJsonString(e.getValue().getFirst())
                                       .getBytes(StandardCharsets.UTF_8)));
 
+                      Object aspectValue =
+                          isAsyncAlternateValidation
+                              ? valueMap.get("value").get("value")
+                              : valueMap.get("value");
+
                       if (e.getValue().getSecond() != null) {
                         return Map.entry(
                             e.getKey(),
                             new GenericAspectV2(
                                 Map.of(
-                                    "systemMetadata", e.getValue().getSecond(),
-                                    "value", valueMap.get("value"))));
+                                    "systemMetadata",
+                                    e.getValue().getSecond(),
+                                    "value",
+                                    aspectValue)));
                       } else {
                         return Map.entry(
-                            e.getKey(),
-                            new GenericAspectV2(Map.of("value", valueMap.get("value"))));
+                            e.getKey(), new GenericAspectV2(Map.of("value", aspectValue)));
                       }
                     } catch (IOException ex) {
                       throw new RuntimeException(ex);
diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityAspectsBodyV3.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityAspectsBodyV3.java
new file mode 100644
index 0000000000000..27107bae87d0e
--- /dev/null
+++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityAspectsBodyV3.java
@@ -0,0 +1,15 @@
+package io.datahubproject.openapi.v3.models;
+
+import java.util.Set;
+import javax.annotation.Nullable;
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.jackson.Jacksonized;
+
+@Data
+@Jacksonized
+@Builder
+public class GenericEntityAspectsBodyV3 {
+  @Nullable private Set<String> entities;
+  @Nullable private Set<String> aspects;
+}
diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityV3.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityV3.java
index 54d6ac2c1736f..ab8c49841c9d9 100644
--- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityV3.java
+++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityV3.java
@@ -29,6 +29,10 @@ public GenericEntityV3(Map<? extends String, ?> m) {
     super(m);
   }
 
+  public String getUrn() {
+    return (String) get("urn");
+  }
+
   @Override
   public Map<String, GenericAspectV3> getAspects() {
     return this.entrySet().stream()
@@ -46,11 +50,12 @@ public GenericEntityV3 build(
                   entry -> {
                     try {
                       String aspectName = entry.getKey();
-                      Map<String, Object> aspectValue =
+                      Map<String, Object> aspectValueMap =
                           objectMapper.readValue(
                               RecordUtils.toJsonString(entry.getValue().getAspect())
                                   .getBytes(StandardCharsets.UTF_8),
                               new TypeReference<>() {});
+
                       Map<String, Object> systemMetadata =
                           entry.getValue().getSystemMetadata() != null
                               ? objectMapper.convertValue(
@@ -65,7 +70,7 @@ public GenericEntityV3 build(
                       return Map.entry(
                           aspectName,
                           GenericAspectV3.builder()
-                              .value(aspectValue)
+                              .value(aspectValueMap)
                               .systemMetadata(systemMetadata)
                               .auditStamp(auditStamp)
                               .build());
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/GlobalControllerExceptionHandler.java
similarity index 81%
rename from metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java
rename to metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/GlobalControllerExceptionHandler.java
index ba0a426fa20e8..c756827cad56b 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/GlobalControllerExceptionHandler.java
@@ -1,9 +1,11 @@
-package io.datahubproject.openapi;
+package io.datahubproject.openapi.config;
 
 import com.linkedin.metadata.dao.throttle.APIThrottleException;
+import io.datahubproject.metadata.exception.ActorAccessException;
 import io.datahubproject.openapi.exception.InvalidUrnException;
 import io.datahubproject.openapi.exception.UnauthorizedException;
 import java.util.Map;
+import javax.annotation.PostConstruct;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.ConversionNotSupportedException;
 import org.springframework.core.Ordered;
@@ -19,6 +21,11 @@
 @ControllerAdvice
 public class GlobalControllerExceptionHandler extends DefaultHandlerExceptionResolver {
 
+  @PostConstruct
+  public void init() {
+    log.info("GlobalControllerExceptionHandler initialized");
+  }
+
   public GlobalControllerExceptionHandler() {
     setOrder(Ordered.HIGHEST_PRECEDENCE);
     setWarnLogCategory(getClass().getName());
@@ -52,4 +59,9 @@ public static ResponseEntity<Map<String, String>> handleUnauthorizedException(
       UnauthorizedException e) {
     return new ResponseEntity<>(Map.of("error", e.getMessage()), HttpStatus.FORBIDDEN);
   }
+
+  @ExceptionHandler(ActorAccessException.class)
+  public static ResponseEntity<Map<String, String>> actorAccessException(ActorAccessException e) {
+    return new ResponseEntity<>(Map.of("error", e.getMessage()), HttpStatus.FORBIDDEN);
+  }
 }
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java
index b7f52e61e9244..592d7bba4211f 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java
@@ -1,5 +1,6 @@
 package io.datahubproject.openapi.controller;
 
+import static com.linkedin.metadata.Constants.TIMESTAMP_MILLIS;
 import static com.linkedin.metadata.authorization.ApiOperation.CREATE;
 import static com.linkedin.metadata.authorization.ApiOperation.DELETE;
 import static com.linkedin.metadata.authorization.ApiOperation.EXISTS;
@@ -32,14 +33,20 @@
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.query.SearchFlags;
+import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.SortCriterion;
 import com.linkedin.metadata.query.filter.SortOrder;
 import com.linkedin.metadata.search.ScrollResult;
 import com.linkedin.metadata.search.SearchEntityArray;
 import com.linkedin.metadata.search.SearchService;
+import com.linkedin.metadata.search.utils.QueryUtils;
+import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import com.linkedin.metadata.utils.AuditStampUtils;
+import com.linkedin.metadata.utils.CriterionUtils;
+import com.linkedin.metadata.utils.GenericRecordUtils;
 import com.linkedin.metadata.utils.SearchUtil;
 import com.linkedin.mxe.SystemMetadata;
+import com.linkedin.timeseries.TimeseriesAspectBase;
 import com.linkedin.util.Pair;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.RequestContext;
@@ -57,6 +64,7 @@
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.http.MediaType;
@@ -83,6 +91,7 @@ public abstract class GenericEntitiesController<
   @Autowired protected EntityRegistry entityRegistry;
   @Autowired protected SearchService searchService;
   @Autowired protected EntityService<?> entityService;
+  @Autowired protected TimeseriesAspectService timeseriesAspectService;
   @Autowired protected AuthorizerChain authorizationChain;
   @Autowired protected ObjectMapper objectMapper;
 
@@ -97,6 +106,7 @@ public abstract class GenericEntitiesController<
    * @param aspectNames the aspect names present
    * @param withSystemMetadata whether to include system metadata in the result
    * @param scrollId the pagination token
+   * @param expandEmpty whether to expand an empty aspects list to all aspects
    * @return result containing entities/aspects
    * @throws URISyntaxException parsing error
    */
@@ -105,24 +115,26 @@ protected abstract S buildScrollResult(
       SearchEntityArray searchEntities,
       Set<String> aspectNames,
       boolean withSystemMetadata,
-      @Nullable String scrollId)
+      @Nullable String scrollId,
+      boolean expandEmpty)
       throws URISyntaxException;
 
   protected List<E> buildEntityList(
       @Nonnull OperationContext opContext,
       List<Urn> urns,
-      Set<String> aspectNames,
-      boolean withSystemMetadata)
+      @Nullable Set<String> aspectNames,
+      boolean withSystemMetadata,
+      boolean expandEmpty)
       throws URISyntaxException {
 
-    LinkedHashMap<Urn, Map<String, Long>> versionMap =
-        resolveAspectNames(
+    LinkedHashMap<Urn, Map<AspectSpec, Long>> aspectSpecMap =
+        resolveAspectSpecs(
             urns.stream()
                 .map(
                     urn ->
                         Map.entry(
                             urn,
-                            aspectNames.stream()
+                            Optional.ofNullable(aspectNames).orElse(Set.of()).stream()
                                 .map(aspectName -> Map.entry(aspectName, 0L))
                                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))))
                 .collect(
@@ -133,24 +145,45 @@ protected List<E> buildEntityList(
                           throw new IllegalStateException("Duplicate key");
                         },
                         LinkedHashMap::new)),
-            0L);
-    return buildEntityVersionedAspectList(opContext, versionMap, withSystemMetadata);
+            0L,
+            expandEmpty);
+
+    return buildEntityVersionedAspectList(
+        opContext, urns, aspectSpecMap, withSystemMetadata, expandEmpty);
   }
 
+  /**
+   * Build a list of entities for an API response
+   *
+   * @param opContext the operation context
+   * @param requestedUrns list of urns requested
+   * @param fetchUrnAspectVersions the map of urn to aspect name and version to fetch
+   * @param withSystemMetadata whether to include system metadata in the response entity
+   * @param expandEmpty whether to expand an empty aspects list to all aspects
+   * @return entity responses
+   * @throws URISyntaxException urn parsing error
+   */
   protected abstract List<E> buildEntityVersionedAspectList(
       @Nonnull OperationContext opContext,
-      LinkedHashMap<Urn, Map<String, Long>> urnAspectVersions,
-      boolean withSystemMetadata)
+      Collection<Urn> requestedUrns,
+      LinkedHashMap<Urn, Map<AspectSpec, Long>> fetchUrnAspectVersions,
+      boolean withSystemMetadata,
+      boolean expandEmpty)
       throws URISyntaxException;
 
   protected abstract List<E> buildEntityList(
-      Set<IngestResult> ingestResults, boolean withSystemMetadata);
+      OperationContext opContext,
+      Collection<IngestResult> ingestResults,
+      boolean withSystemMetadata);
 
   protected abstract E buildGenericEntity(
       @Nonnull String aspectName,
       @Nonnull UpdateAspectResult updateAspectResult,
       boolean withSystemMetadata);
 
+  protected abstract E buildGenericEntity(
+      @Nonnull String aspectName, @Nonnull IngestResult ingestResult, boolean withSystemMetadata);
+
   protected abstract AspectsBatch toMCPBatch(
       @Nonnull OperationContext opContext, String entityArrayList, Actor actor)
       throws JsonProcessingException, InvalidUrnException;
@@ -166,7 +199,8 @@ public ResponseEntity<S> getEntities(
       @RequestParam(value = "count", defaultValue = "10") Integer count,
       @RequestParam(value = "query", defaultValue = "*") String query,
       @RequestParam(value = "scrollId", required = false) String scrollId,
-      @RequestParam(value = "sort", required = false, defaultValue = "urn") String sortField,
+      @RequestParam(value = "sort", required = false, defaultValue = "urn") @Deprecated
+          String sortField,
       @RequestParam(value = "sortCriteria", required = false) List<String> sortFields,
       @RequestParam(value = "sortOrder", required = false, defaultValue = "ASCENDING")
           String sortOrder,
@@ -175,7 +209,9 @@ public ResponseEntity<S> getEntities(
       @RequestParam(value = "skipCache", required = false, defaultValue = "false")
           Boolean skipCache,
       @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false")
-          Boolean includeSoftDelete)
+          Boolean includeSoftDelete,
+      @RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m")
+          String pitKeepALive)
       throws URISyntaxException {
 
     EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName);
@@ -196,14 +232,20 @@ public ResponseEntity<S> getEntities(
           authentication.getActor().toUrnStr() + " is unauthorized to " + READ + "  entities.");
     }
 
+    SortOrder finalSortOrder =
+        SortOrder.valueOf(Optional.ofNullable(sortOrder).orElse("ASCENDING"));
+
     List<SortCriterion> sortCriteria;
-    if (!CollectionUtils.isEmpty(sortFields)) {
+    if (!CollectionUtils.isEmpty(sortFields)
+        && sortFields.stream().anyMatch(StringUtils::isNotBlank)) {
       sortCriteria = new ArrayList<>();
-      sortFields.forEach(
-          field -> sortCriteria.add(SearchUtil.sortBy(field, SortOrder.valueOf(sortOrder))));
+      sortFields.stream()
+          .filter(StringUtils::isNotBlank)
+          .forEach(field -> sortCriteria.add(SearchUtil.sortBy(field, finalSortOrder)));
+    } else if (StringUtils.isNotBlank(sortField)) {
+      sortCriteria = Collections.singletonList(SearchUtil.sortBy(sortField, finalSortOrder));
     } else {
-      sortCriteria =
-          Collections.singletonList(SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder)));
+      sortCriteria = Collections.singletonList(SearchUtil.sortBy("urn", finalSortOrder));
     }
 
     ScrollResult result =
@@ -217,7 +259,7 @@ public ResponseEntity<S> getEntities(
             null,
             sortCriteria,
             scrollId,
-            null,
+            pitKeepALive,
             count);
 
     if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) {
@@ -225,13 +267,17 @@ public ResponseEntity<S> getEntities(
           authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities.");
     }
 
+    Set<String> mergedAspects =
+        ImmutableSet.<String>builder().addAll(aspects1).addAll(aspects2).build();
+
     return ResponseEntity.ok(
         buildScrollResult(
             opContext,
             result.getEntities(),
-            ImmutableSet.<String>builder().addAll(aspects1).addAll(aspects2).build(),
+            mergedAspects,
             withSystemMetadata,
-            result.getScrollId()));
+            result.getScrollId(),
+            true));
   }
 
   @Tag(name = "Generic Entities")
@@ -269,7 +315,8 @@ public ResponseEntity<E> getEntity(
             opContext,
             List.of(urn),
             ImmutableSet.<String>builder().addAll(aspects1).addAll(aspects2).build(),
-            withSystemMetadata)
+            withSystemMetadata,
+            true)
         .stream()
         .findFirst()
         .map(ResponseEntity::ok)
@@ -344,13 +391,17 @@ public ResponseEntity<A> getAspect(
 
     final List<E> resultList;
     if (version == 0) {
-      resultList = buildEntityList(opContext, List.of(urn), Set.of(aspectName), withSystemMetadata);
+      resultList =
+          buildEntityList(opContext, List.of(urn), Set.of(aspectName), withSystemMetadata, true);
     } else {
       resultList =
           buildEntityVersionedAspectList(
               opContext,
-              new LinkedHashMap<>(Map.of(urn, Map.of(aspectName, version))),
-              withSystemMetadata);
+              List.of(urn),
+              resolveAspectSpecs(
+                  new LinkedHashMap<>(Map.of(urn, Map.of(aspectName, version))), 0L, true),
+              withSystemMetadata,
+              true);
     }
 
     return resultList.stream()
@@ -395,9 +446,10 @@ public ResponseEntity<Object> headAspect(
           authentication.getActor().toUrnStr() + " is unauthorized to " + EXISTS + " entities.");
     }
 
-    return exists(opContext, urn, lookupAspectSpec(urn, aspectName).getName(), includeSoftDelete)
-        ? ResponseEntity.noContent().build()
-        : ResponseEntity.notFound().build();
+    return lookupAspectSpec(urn, aspectName)
+        .filter(aspectSpec -> exists(opContext, urn, aspectSpec.getName(), includeSoftDelete))
+        .map(aspectSpec -> ResponseEntity.noContent().build())
+        .orElse(ResponseEntity.notFound().build());
   }
 
   @Tag(name = "Generic Entities")
@@ -443,7 +495,7 @@ public void deleteEntity(
       entityService.deleteUrn(opContext, urn);
     } else {
       aspects.stream()
-          .map(aspectName -> lookupAspectSpec(urn, aspectName).getName())
+          .map(aspectName -> lookupAspectSpec(urn, aspectName).get().getName())
           .forEach(
               aspectName ->
                   entityService.deleteAspect(opContext, entityUrn, aspectName, Map.of(), true));
@@ -479,12 +531,13 @@ public ResponseEntity<List<E>> createEntity(
     }
 
     AspectsBatch batch = toMCPBatch(opContext, jsonEntityList, authentication.getActor());
-    Set<IngestResult> results = entityService.ingestProposal(opContext, batch, async);
+    List<IngestResult> results = entityService.ingestProposal(opContext, batch, async);
 
     if (!async) {
-      return ResponseEntity.ok(buildEntityList(results, withSystemMetadata));
+      return ResponseEntity.ok(buildEntityList(opContext, results, withSystemMetadata));
     } else {
-      return ResponseEntity.accepted().body(buildEntityList(results, withSystemMetadata));
+      return ResponseEntity.accepted()
+          .body(buildEntityList(opContext, results, withSystemMetadata));
     }
   }
 
@@ -515,8 +568,32 @@ public void deleteAspect(
           authentication.getActor().toUrnStr() + " is unauthorized to " + DELETE + " entities.");
     }
 
-    entityService.deleteAspect(
-        opContext, entityUrn, lookupAspectSpec(urn, aspectName).getName(), Map.of(), true);
+    lookupAspectSpec(urn, aspectName)
+        .ifPresent(
+            aspectSpec -> {
+              if (aspectSpec.isTimeseries()) {
+                Map<Urn, Map<String, com.linkedin.metadata.aspect.EnvelopedAspect>> latestMap =
+                    timeseriesAspectService.getLatestTimeseriesAspectValues(
+                        opContext, Set.of(urn), Set.of(aspectSpec.getName()), null);
+                com.linkedin.metadata.aspect.EnvelopedAspect latestAspect =
+                    latestMap.getOrDefault(urn, Map.of()).get(aspectSpec.getName());
+                if (latestAspect != null) {
+                  Long latestTs =
+                      new TimeseriesAspectBase(toRecordTemplate(aspectSpec, latestAspect).data())
+                          .getTimestampMillis();
+                  timeseriesAspectService.deleteAspectValues(
+                      opContext,
+                      urn.getEntityType(),
+                      aspectSpec.getName(),
+                      QueryUtils.newFilter(
+                          CriterionUtils.buildCriterion(
+                              TIMESTAMP_MILLIS, Condition.EQUAL, String.valueOf(latestTs))));
+                }
+              } else {
+                entityService.deleteAspect(
+                    opContext, entityUrn, aspectSpec.getName(), Map.of(), true);
+              }
+            });
   }
 
   @Tag(name = "Generic Aspects")
@@ -529,8 +606,11 @@ public ResponseEntity<E> createAspect(
       @PathVariable("entityName") String entityName,
       @PathVariable("entityUrn") String entityUrn,
       @PathVariable("aspectName") String aspectName,
+      @RequestParam(value = "async", required = false, defaultValue = "false") Boolean async,
       @RequestParam(value = "systemMetadata", required = false, defaultValue = "false")
           Boolean withSystemMetadata,
+      @RequestParam(value = "createIfEntityNotExists", required = false, defaultValue = "false")
+          Boolean createIfEntityNotExists,
       @RequestParam(value = "createIfNotExists", required = false, defaultValue = "true")
           Boolean createIfNotExists,
       @RequestBody @Nonnull String jsonAspect)
@@ -554,30 +634,45 @@ public ResponseEntity<E> createAspect(
           authentication.getActor().toUrnStr() + " is unauthorized to " + CREATE + " entities.");
     }
 
-    AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName);
+    AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName).get();
     ChangeMCP upsert =
         toUpsertItem(
-            opContext.getRetrieverContext().get().getAspectRetriever(),
+            opContext.getRetrieverContext().getAspectRetriever(),
             urn,
             aspectSpec,
+            createIfEntityNotExists,
             createIfNotExists,
             jsonAspect,
             authentication.getActor());
 
-    List<UpdateAspectResult> results =
-        entityService.ingestAspects(
+    List<IngestResult> results =
+        entityService.ingestProposal(
             opContext,
             AspectsBatchImpl.builder()
-                .retrieverContext(opContext.getRetrieverContext().get())
+                .retrieverContext(opContext.getRetrieverContext())
                 .items(List.of(upsert))
                 .build(),
-            true,
-            true);
+            async);
 
-    return ResponseEntity.of(
-        results.stream()
-            .findFirst()
-            .map(result -> buildGenericEntity(aspectName, result, withSystemMetadata)));
+    if (!async) {
+      return ResponseEntity.of(
+          results.stream()
+              .filter(item -> aspectSpec.getName().equals(item.getRequest().getAspectName()))
+              .findFirst()
+              .map(
+                  result ->
+                      buildGenericEntity(
+                          aspectSpec.getName(), result.getResult(), withSystemMetadata)));
+    } else {
+      return results.stream()
+          .filter(item -> aspectSpec.getName().equals(item.getRequest().getAspectName()))
+          .map(
+              result ->
+                  ResponseEntity.accepted()
+                      .body(buildGenericEntity(aspectSpec.getName(), result, withSystemMetadata)))
+          .findFirst()
+          .orElse(ResponseEntity.accepted().build());
+    }
   }
 
   @Tag(name = "Generic Aspects")
@@ -618,7 +713,7 @@ public ResponseEntity<E> patchAspect(
           authentication.getActor().toUrnStr() + " is unauthorized to " + UPDATE + " entities.");
     }
 
-    AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName);
+    AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName).get();
     RecordTemplate currentValue = entityService.getAspect(opContext, urn, aspectSpec.getName(), 0);
 
     GenericPatchTemplate<? extends RecordTemplate> genericPatchTemplate =
@@ -630,7 +725,7 @@ public ResponseEntity<E> patchAspect(
             .build();
     ChangeMCP upsert =
         toUpsertItem(
-            opContext.getRetrieverContext().get().getAspectRetriever(),
+            opContext.getRetrieverContext().getAspectRetriever(),
             validatedUrn(entityUrn),
             aspectSpec,
             currentValue,
@@ -641,7 +736,7 @@ public ResponseEntity<E> patchAspect(
         entityService.ingestAspects(
             opContext,
             AspectsBatchImpl.builder()
-                .retrieverContext(opContext.getRetrieverContext().get())
+                .retrieverContext(opContext.getRetrieverContext())
                 .items(List.of(upsert))
                 .build(),
             true,
@@ -672,20 +767,22 @@ protected Boolean exists(
    *
    * @param requestedAspectNames requested aspects
    * @param <T> map values
+   * @param expandEmpty whether to expand empty aspect names to all aspect names
    * @return updated map
    */
-  protected <T> LinkedHashMap<Urn, Map<String, T>> resolveAspectNames(
-      LinkedHashMap<Urn, Map<String, T>> requestedAspectNames, @Nonnull T defaultValue) {
+  protected <T> LinkedHashMap<Urn, Map<AspectSpec, T>> resolveAspectSpecs(
+      LinkedHashMap<Urn, Map<String, T>> requestedAspectNames,
+      @Nonnull T defaultValue,
+      boolean expandEmpty) {
     return requestedAspectNames.entrySet().stream()
         .map(
             entry -> {
               final Urn urn = entry.getKey();
-              if (entry.getValue().isEmpty() || entry.getValue().containsKey("")) {
+              if (expandEmpty && (entry.getValue().isEmpty() || entry.getValue().containsKey(""))) {
                 // All aspects specified
-                Set<String> allNames =
-                    entityRegistry.getEntitySpec(urn.getEntityType()).getAspectSpecs().stream()
-                        .map(AspectSpec::getName)
-                        .collect(Collectors.toSet());
+                Set<AspectSpec> allNames =
+                    new HashSet<>(
+                        entityRegistry.getEntitySpec(urn.getEntityType()).getAspectSpecs());
                 return Map.entry(
                     urn,
                     allNames.stream()
@@ -694,15 +791,15 @@ protected <T> LinkedHashMap<Urn, Map<String, T>> resolveAspectNames(
                                 Map.entry(
                                     aspectName, entry.getValue().getOrDefault("", defaultValue)))
                         .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
-              } else {
-                final Map<String, String> normalizedNames =
+              } else if (!entry.getValue().keySet().isEmpty()) {
+                final Map<String, AspectSpec> normalizedNames =
                     entry.getValue().keySet().stream()
                         .map(
                             requestAspectName ->
                                 Map.entry(
-                                    requestAspectName,
-                                    lookupAspectSpec(urn, requestAspectName).getName()))
-                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                                    requestAspectName, lookupAspectSpec(urn, requestAspectName)))
+                        .filter(aspectSpecEntry -> aspectSpecEntry.getValue().isPresent())
+                        .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get()));
                 return Map.entry(
                     urn,
                     entry.getValue().entrySet().stream()
@@ -712,8 +809,32 @@ protected <T> LinkedHashMap<Urn, Map<String, T>> resolveAspectNames(
                                 Map.entry(
                                     normalizedNames.get(reqEntry.getKey()), reqEntry.getValue()))
                         .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+              } else {
+                return (Map.Entry<Urn, Map<AspectSpec, T>>) null;
               }
             })
+        .filter(Objects::nonNull)
+        .collect(
+            Collectors.toMap(
+                Map.Entry::getKey,
+                Map.Entry::getValue,
+                (a, b) -> {
+                  throw new IllegalStateException("Duplicate key");
+                },
+                LinkedHashMap::new));
+  }
+
+  protected static <T> LinkedHashMap<Urn, Map<String, T>> aspectSpecsToAspectNames(
+      LinkedHashMap<Urn, Map<AspectSpec, T>> urnAspectSpecsMap, boolean timeseries) {
+    return urnAspectSpecsMap.entrySet().stream()
+        .map(
+            e ->
+                Map.entry(
+                    e.getKey(),
+                    e.getValue().entrySet().stream()
+                        .filter(a -> timeseries == a.getKey().isTimeseries())
+                        .map(a -> Map.entry(a.getKey().getName(), a.getValue()))
+                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))))
         .collect(
             Collectors.toMap(
                 Map.Entry::getKey,
@@ -732,12 +853,12 @@ protected Map<String, Pair<RecordTemplate, SystemMetadata>> toAspectMap(
                 Map.entry(
                     a.getName(),
                     Pair.of(
-                        toRecordTemplate(lookupAspectSpec(urn, a.getName()), a),
+                        toRecordTemplate(lookupAspectSpec(urn, a.getName()).get(), a),
                         withSystemMetadata ? a.getSystemMetadata() : null)))
         .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
   }
 
-  protected AspectSpec lookupAspectSpec(Urn urn, String aspectName) {
+  protected Optional<AspectSpec> lookupAspectSpec(Urn urn, String aspectName) {
     return lookupAspectSpec(entityRegistry.getEntitySpec(urn.getEntityType()), aspectName);
   }
 
@@ -747,10 +868,19 @@ protected RecordTemplate toRecordTemplate(
         aspectSpec.getDataTemplateClass(), envelopedAspect.getValue().data());
   }
 
+  protected RecordTemplate toRecordTemplate(
+      AspectSpec aspectSpec, com.linkedin.metadata.aspect.EnvelopedAspect envelopedAspect) {
+    return GenericRecordUtils.deserializeAspect(
+        envelopedAspect.getAspect().getValue(),
+        envelopedAspect.getAspect().getContentType(),
+        aspectSpec);
+  }
+
   protected abstract ChangeMCP toUpsertItem(
       @Nonnull AspectRetriever aspectRetriever,
       Urn entityUrn,
       AspectSpec aspectSpec,
+      Boolean createIfEntityNotExists,
       Boolean createIfNotExists,
       String jsonAspect,
       Actor actor)
@@ -777,13 +907,16 @@ protected ChangeMCP toUpsertItem(
    *
    * @return
    */
-  protected static AspectSpec lookupAspectSpec(EntitySpec entitySpec, String aspectName) {
+  protected static Optional<AspectSpec> lookupAspectSpec(EntitySpec entitySpec, String aspectName) {
+    if (entitySpec == null) {
+      return Optional.empty();
+    }
+
     return entitySpec.getAspectSpec(aspectName) != null
-        ? entitySpec.getAspectSpec(aspectName)
+        ? Optional.of(entitySpec.getAspectSpec(aspectName))
         : entitySpec.getAspectSpecs().stream()
             .filter(aspec -> aspec.getName().toLowerCase().equals(aspectName))
-            .findFirst()
-            .get();
+            .findFirst();
   }
 
   protected static Urn validatedUrn(String urn) throws InvalidUrnException {
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/test/IdController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/test/IdController.java
new file mode 100644
index 0000000000000..99d3879ab9a32
--- /dev/null
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/test/IdController.java
@@ -0,0 +1,54 @@
+package io.datahubproject.openapi.operations.test;
+
+import com.datahub.authentication.Authentication;
+import com.datahub.authentication.AuthenticationContext;
+import com.datahub.authorization.AuthorizerChain;
+import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.metadata.context.RequestContext;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.servlet.http.HttpServletRequest;
+import java.util.List;
+import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/operations/identity")
+@Slf4j
+@Tag(name = "Identity", description = "An API for checking identity")
+public class IdController {
+  private final AuthorizerChain authorizerChain;
+  private final OperationContext systemOperationContext;
+
+  public IdController(OperationContext systemOperationContext, AuthorizerChain authorizerChain) {
+    this.systemOperationContext = systemOperationContext;
+    this.authorizerChain = authorizerChain;
+  }
+
+  @Tag(name = "User")
+  @GetMapping(path = "/user/urn", produces = MediaType.APPLICATION_JSON_VALUE)
+  @Operation(summary = "User id")
+  public ResponseEntity<Map<String, String>> getUserId(
+      HttpServletRequest request,
+      @RequestParam(value = "skipCache", required = false, defaultValue = "false")
+          Boolean skipCache) {
+    Authentication authentication = AuthenticationContext.getAuthentication();
+    String actorUrnStr = authentication.getActor().toUrnStr();
+
+    OperationContext.asSession(
+        systemOperationContext,
+        RequestContext.builder().buildOpenapi(actorUrnStr, request, "getUserIdentity", List.of()),
+        authorizerChain,
+        authentication,
+        true,
+        skipCache);
+
+    return ResponseEntity.ok(Map.of("urn", actorUrnStr));
+  }
+}
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java
index c38f2db0eefbb..ca425810c87a0 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java
@@ -491,7 +491,7 @@ public static List<Pair<String, Boolean>> ingestBatchProposal(
     try {
       AspectsBatch batch =
           AspectsBatchImpl.builder()
-              .mcps(serviceProposals, auditStamp, opContext.getRetrieverContext().get())
+              .mcps(serviceProposals, auditStamp, opContext.getRetrieverContext())
               .build();
 
       Map<Urn, List<IngestResult>> resultMap =
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java
index d20acbee79b22..b1c5709ef0147 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java
@@ -23,11 +23,14 @@
 import com.linkedin.metadata.entity.UpdateAspectResult;
 import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
 import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
+import com.linkedin.metadata.entity.ebean.batch.ProposedItem;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchEntityArray;
 import com.linkedin.metadata.utils.AuditStampUtils;
 import com.linkedin.metadata.utils.GenericRecordUtils;
+import com.linkedin.metadata.utils.SystemMetadataUtils;
+import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.mxe.SystemMetadata;
 import com.linkedin.util.Pair;
 import io.datahubproject.metadata.context.OperationContext;
@@ -46,6 +49,7 @@
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -114,7 +118,8 @@ public ResponseEntity<BatchGetUrnResponseV2<GenericAspectV2, GenericEntityV2>> g
                             opContext,
                             urns,
                             new HashSet<>(request.getAspectNames()),
-                            request.isWithSystemMetadata())))
+                            request.isWithSystemMetadata(),
+                            true)))
                 .build()));
   }
 
@@ -124,10 +129,12 @@ public GenericEntityScrollResultV2 buildScrollResult(
       SearchEntityArray searchEntities,
       Set<String> aspectNames,
       boolean withSystemMetadata,
-      @Nullable String scrollId)
+      @Nullable String scrollId,
+      boolean expandEmpty)
       throws URISyntaxException {
     return GenericEntityScrollResultV2.builder()
-        .results(toRecordTemplates(opContext, searchEntities, aspectNames, withSystemMetadata))
+        .results(
+            toRecordTemplates(opContext, searchEntities, aspectNames, withSystemMetadata, true))
         .scrollId(scrollId)
         .build();
   }
@@ -155,9 +162,28 @@ protected AspectsBatch toMCPBatch(
         while (aspectItr.hasNext()) {
           Map.Entry<String, JsonNode> aspect = aspectItr.next();
 
-          AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey());
+          AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()).get();
+          JsonNode jsonNodeAspect = aspect.getValue().get("value");
 
-          if (aspectSpec != null) {
+          if (opContext.getValidationContext().isAlternateValidation()) {
+            ProposedItem.ProposedItemBuilder builder =
+                ProposedItem.builder()
+                    .metadataChangeProposal(
+                        new MetadataChangeProposal()
+                            .setEntityUrn(entityUrn)
+                            .setAspectName(aspect.getKey())
+                            .setEntityType(entityUrn.getEntityType())
+                            .setChangeType(ChangeType.UPSERT)
+                            .setAspect(GenericRecordUtils.serializeAspect(jsonNodeAspect))
+                            .setSystemMetadata(SystemMetadataUtils.createDefaultSystemMetadata()))
+                    .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
+                    .entitySpec(
+                        opContext
+                            .getAspectRetriever()
+                            .getEntityRegistry()
+                            .getEntitySpec(entityUrn.getEntityType()));
+            items.add(builder.build());
+          } else if (aspectSpec != null) {
             ChangeItemImpl.ChangeItemImplBuilder builder =
                 ChangeItemImpl.builder()
                     .urn(entityUrn)
@@ -166,7 +192,7 @@ protected AspectsBatch toMCPBatch(
                     .recordTemplate(
                         GenericRecordUtils.deserializeAspect(
                             ByteString.copyString(
-                                objectMapper.writeValueAsString(aspect.getValue().get("value")),
+                                objectMapper.writeValueAsString(jsonNodeAspect),
                                 StandardCharsets.UTF_8),
                             GenericRecordUtils.JSON,
                             aspectSpec));
@@ -177,7 +203,7 @@ protected AspectsBatch toMCPBatch(
                       objectMapper.writeValueAsString(aspect.getValue().get("systemMetadata"))));
             }
 
-            items.add(builder.build(opContext.getRetrieverContext().get().getAspectRetriever()));
+            items.add(builder.build(opContext.getAspectRetriever()));
           }
         }
       }
@@ -185,19 +211,22 @@ protected AspectsBatch toMCPBatch(
 
     return AspectsBatchImpl.builder()
         .items(items)
-        .retrieverContext(opContext.getRetrieverContext().get())
+        .retrieverContext(opContext.getRetrieverContext())
         .build();
   }
 
   @Override
   protected List<GenericEntityV2> buildEntityVersionedAspectList(
       @Nonnull OperationContext opContext,
-      LinkedHashMap<Urn, Map<String, Long>> urnAspectVersions,
-      boolean withSystemMetadata)
+      Collection<Urn> requestedUrns,
+      LinkedHashMap<Urn, Map<AspectSpec, Long>> urnAspectVersions,
+      boolean withSystemMetadata,
+      boolean expandEmpty)
       throws URISyntaxException {
+
     Map<Urn, List<EnvelopedAspect>> aspects =
         entityService.getEnvelopedVersionedAspects(
-            opContext, resolveAspectNames(urnAspectVersions, 0L), true);
+            opContext, aspectSpecsToAspectNames(urnAspectVersions, false), true);
 
     return urnAspectVersions.keySet().stream()
         .map(
@@ -226,22 +255,39 @@ protected GenericEntityV2 buildGenericEntity(
                     withSystemMetadata ? updateAspectResult.getNewSystemMetadata() : null)));
   }
 
+  @Override
+  protected GenericEntityV2 buildGenericEntity(
+      @Nonnull String aspectName, @Nonnull IngestResult ingestResult, boolean withSystemMetadata) {
+    return GenericEntityV2.builder()
+        .urn(ingestResult.getUrn().toString())
+        .build(
+            objectMapper,
+            Map.of(
+                aspectName,
+                Pair.of(
+                    ingestResult.getRequest().getRecordTemplate(),
+                    withSystemMetadata ? ingestResult.getRequest().getSystemMetadata() : null)));
+  }
+
   private List<GenericEntityV2> toRecordTemplates(
       @Nonnull OperationContext opContext,
       SearchEntityArray searchEntities,
       Set<String> aspectNames,
-      boolean withSystemMetadata)
+      boolean withSystemMetadata,
+      boolean expandEmpty)
       throws URISyntaxException {
     return buildEntityList(
         opContext,
         searchEntities.stream().map(SearchEntity::getEntity).collect(Collectors.toList()),
         aspectNames,
-        withSystemMetadata);
+        withSystemMetadata,
+        true);
   }
 
-  @Override
   protected List<GenericEntityV2> buildEntityList(
-      Set<IngestResult> ingestResults, boolean withSystemMetadata) {
+      OperationContext opContext,
+      Collection<IngestResult> ingestResults,
+      boolean withSystemMetadata) {
     List<GenericEntityV2> responseList = new LinkedList<>();
 
     Map<Urn, List<IngestResult>> entityMap =
@@ -260,7 +306,10 @@ protected List<GenericEntityV2> buildEntityList(
       responseList.add(
           GenericEntityV2.builder()
               .urn(urnAspects.getKey().toString())
-              .build(objectMapper, aspectsMap));
+              .build(
+                  objectMapper,
+                  aspectsMap,
+                  opContext.getValidationContext().isAlternateValidation()));
     }
     return responseList;
   }
@@ -270,14 +319,25 @@ protected ChangeMCP toUpsertItem(
       @Nonnull AspectRetriever aspectRetriever,
       Urn entityUrn,
       AspectSpec aspectSpec,
+      Boolean createIfEntityNotExists,
       Boolean createIfNotExists,
       String jsonAspect,
       Actor actor)
       throws URISyntaxException {
+
+    final ChangeType changeType;
+    if (Boolean.TRUE.equals(createIfEntityNotExists)) {
+      changeType = ChangeType.CREATE_ENTITY;
+    } else if (Boolean.TRUE.equals(createIfNotExists)) {
+      changeType = ChangeType.CREATE;
+    } else {
+      changeType = ChangeType.UPSERT;
+    }
+
     return ChangeItemImpl.builder()
         .urn(entityUrn)
         .aspectName(aspectSpec.getName())
-        .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT)
+        .changeType(changeType)
         .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
         .recordTemplate(
             GenericRecordUtils.deserializeAspect(
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java
index 1b3dcfc94f4f2..c6b8d579d879e 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java
@@ -45,6 +45,7 @@ public class OpenAPIV3Generator {
   private static final String NAME_VERSION = "version";
   private static final String NAME_SCROLL_ID = "scrollId";
   private static final String NAME_INCLUDE_SOFT_DELETE = "includeSoftDelete";
+  private static final String NAME_PIT_KEEP_ALIVE = "pitKeepAlive";
   private static final String PROPERTY_VALUE = "value";
   private static final String PROPERTY_URN = "urn";
   private static final String PROPERTY_PATCH = "patch";
@@ -60,6 +61,8 @@ public class OpenAPIV3Generator {
   private static final String ENTITY_REQUEST_SUFFIX = "Entity" + REQUEST_SUFFIX;
   private static final String ENTITY_RESPONSE_SUFFIX = "Entity" + RESPONSE_SUFFIX;
   private static final String NAME_SKIP_CACHE = "skipCache";
+  private static final String ASPECTS = "Aspects";
+  private static final String ENTITIES = "Entities";
 
   public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) {
     final Set<String> aspectNames = entityRegistry.getAspectSpecs().keySet();
@@ -75,10 +78,19 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) {
     info.setTitle("Entity API");
     info.setDescription("This is a service for DataHub Entities.");
     info.setVersion("v3");
+
     // Components
     final Components components = new Components();
+
+    // Cross-entity components
+    components.addSchemas(
+        ENTITIES + ENTITY_REQUEST_SUFFIX, buildEntitiesRequestSchema(entityRegistry, aspectNames));
+    components.addSchemas(
+        ENTITIES + ENTITY_RESPONSE_SUFFIX, buildEntitySchema(entityRegistry, aspectNames, true));
+    components.addSchemas(
+        "Scroll" + ENTITIES + ENTITY_RESPONSE_SUFFIX, buildEntitiesScrollSchema());
+
     // --> Aspect components
-    components.addSchemas("SortOrder", new Schema()._enum(List.of("ASCENDING", "DESCENDING")));
     components.addSchemas("AspectPatch", buildAspectPatchSchema());
     components.addSchemas(
         "BatchGetRequestBody",
@@ -94,6 +106,10 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) {
                         .description("System headers for the operation.")
                         .nullable(true)))
             .nullable(true));
+
+    // --> Aspect components
+    components.addSchemas(
+        ASPECTS + ASPECT_RESPONSE_SUFFIX, buildAspectsRefResponseSchema(entityRegistry));
     entityRegistry
         .getAspectSpecs()
         .values()
@@ -108,6 +124,7 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) {
                   upperAspectName + ASPECT_RESPONSE_SUFFIX,
                   buildAspectRefResponseSchema(upperAspectName));
             });
+
     // --> Entity components
     entityRegistry.getEntitySpecs().values().stream()
         .filter(e -> aspectNames.contains(e.getKeyAspectName()))
@@ -124,18 +141,34 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) {
                   "BatchGet" + entityName + ENTITY_REQUEST_SUFFIX,
                   buildEntityBatchGetRequestSchema(e, aspectNames));
             });
+
+    components.addSchemas("SortOrder", new Schema()._enum(List.of("ASCENDING", "DESCENDING")));
+    // TODO: Correct handling of SystemMetadata and AuditStamp
+    components.addSchemas(
+        "SystemMetadata", new Schema().type(TYPE_OBJECT).additionalProperties(true));
+    components.addSchemas("AuditStamp", new Schema().type(TYPE_OBJECT).additionalProperties(true));
+
     // Parameters
+
+    // --> Entity Parameters
     entityRegistry.getEntitySpecs().values().stream()
         .filter(e -> definitionNames.contains(e.getKeyAspectName()))
         .forEach(
             e -> {
-              final String parameterName = toUpperFirst(e.getName()) + "Aspects";
+              final String parameterName = toUpperFirst(e.getName()) + ASPECTS;
               components.addParameters(
                   parameterName + MODEL_VERSION, buildParameterSchema(e, definitionNames));
             });
+
     addExtraParameters(components);
+
     // Path
     final Paths paths = new Paths();
+
+    // --> Cross-entity Paths
+    paths.addPathItem("/v3/entity/scroll", buildGenericListEntitiesPath());
+
+    // --> Entity Paths
     entityRegistry.getEntitySpecs().values().stream()
         .filter(e -> definitionNames.contains(e.getName()))
         .sorted(Comparator.comparing(EntitySpec::getName))
@@ -151,6 +184,8 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) {
                   String.format("/v3/entity/%s/{urn}", e.getName().toLowerCase()),
                   buildSingleEntityPath(e));
             });
+
+    // --> Aspect Paths
     entityRegistry.getEntitySpecs().values().stream()
         .filter(e -> definitionNames.contains(e.getName()))
         .sorted(Comparator.comparing(EntitySpec::getName))
@@ -165,19 +200,14 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) {
                               String.format(
                                   "/v3/entity/%s/{urn}/%s",
                                   e.getName().toLowerCase(), a.getName().toLowerCase()),
-                              buildSingleEntityAspectPath(
-                                  e, a.getName(), a.getPegasusSchema().getName())));
+                              buildSingleEntityAspectPath(e, a)));
             });
-    // TODO: Correct handling of SystemMetadata and AuditStamp
-    components.addSchemas(
-        "SystemMetadata", new Schema().type(TYPE_OBJECT).additionalProperties(true));
-    components.addSchemas("AuditStamp", new Schema().type(TYPE_OBJECT).additionalProperties(true));
     return new OpenAPI().openapi("3.0.1").info(info).paths(paths).components(components);
   }
 
   private static PathItem buildSingleEntityPath(final EntitySpec entity) {
     final String upperFirst = toUpperFirst(entity.getName());
-    final String aspectParameterName = upperFirst + "Aspects";
+    final String aspectParameterName = upperFirst + ASPECTS;
     final PathItem result = new PathItem();
 
     // Get Operation
@@ -280,7 +310,7 @@ private static PathItem buildSingleEntityPath(final EntitySpec entity) {
 
   private static PathItem buildListEntityPath(final EntitySpec entity) {
     final String upperFirst = toUpperFirst(entity.getName());
-    final String aspectParameterName = upperFirst + "Aspects";
+    final String aspectParameterName = upperFirst + ASPECTS;
     final PathItem result = new PathItem();
     final List<Parameter> parameters =
         List.of(
@@ -327,7 +357,8 @@ private static PathItem buildListEntityPath(final EntitySpec entity) {
             .summary(String.format("Scroll/List %s.", upperFirst))
             .parameters(parameters)
             .tags(List.of(entity.getName() + " Entity"))
-            .description("Scroll indexed entities. Will not include soft deleted entities.")
+            .description(
+                "Scroll indexed entities. Will not include soft deleted entities by default.")
             .responses(new ApiResponses().addApiResponse("200", successApiResponse)));
 
     // Post Operation
@@ -452,6 +483,80 @@ private static PathItem buildBatchGetEntityPath(final EntitySpec entity) {
     return result;
   }
 
+  private static PathItem buildGenericListEntitiesPath() {
+    final PathItem result = new PathItem();
+    final List<Parameter> parameters =
+        List.of(
+            new Parameter()
+                .in(NAME_QUERY)
+                .name(NAME_SYSTEM_METADATA)
+                .description("Include systemMetadata with response.")
+                .schema(new Schema().type(TYPE_BOOLEAN)._default(false)),
+            new Parameter()
+                .in(NAME_QUERY)
+                .name(NAME_INCLUDE_SOFT_DELETE)
+                .description("Include soft-deleted aspects with response.")
+                .schema(new Schema().type(TYPE_BOOLEAN)._default(false)),
+            new Parameter()
+                .in(NAME_QUERY)
+                .name(NAME_SKIP_CACHE)
+                .description("Skip cache when listing entities.")
+                .schema(new Schema().type(TYPE_BOOLEAN)._default(false)),
+            new Parameter()
+                .in(NAME_QUERY)
+                .name(NAME_PIT_KEEP_ALIVE)
+                .description(
+                    "Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.")
+                .schema(new Schema().type(TYPE_STRING)._default("5m")),
+            new Parameter().$ref("#/components/parameters/PaginationCount" + MODEL_VERSION),
+            new Parameter().$ref("#/components/parameters/ScrollId" + MODEL_VERSION),
+            new Parameter().$ref("#/components/parameters/SortBy" + MODEL_VERSION),
+            new Parameter().$ref("#/components/parameters/SortOrder" + MODEL_VERSION),
+            new Parameter().$ref("#/components/parameters/ScrollQuery" + MODEL_VERSION));
+    final ApiResponse successApiResponse =
+        new ApiResponse()
+            .description("Success")
+            .content(
+                new Content()
+                    .addMediaType(
+                        "application/json",
+                        new MediaType()
+                            .schema(
+                                new Schema()
+                                    .$ref(
+                                        String.format(
+                                            "#/components/schemas/Scroll%s%s",
+                                            ENTITIES, ENTITY_RESPONSE_SUFFIX)))));
+
+    final RequestBody requestBody =
+        new RequestBody()
+            .description(
+                "Scroll entities and aspects. If the `aspects` list is not specified then NO aspects will be returned. If the `aspects` list is emtpy, all aspects will be returned.")
+            .required(false)
+            .content(
+                new Content()
+                    .addMediaType(
+                        "application/json",
+                        new MediaType()
+                            .schema(
+                                new Schema()
+                                    .$ref(
+                                        String.format(
+                                            "#/components/schemas/%s%s",
+                                            ENTITIES, ENTITY_REQUEST_SUFFIX)))));
+
+    result.setPost(
+        new Operation()
+            .summary(String.format("Scroll/List %s.", ENTITIES))
+            .parameters(parameters)
+            .tags(List.of("Generic Entities"))
+            .description("Scroll indexed entities. Will not include soft deleted entities.")
+            .requestBody(requestBody)
+            .responses(new ApiResponses().addApiResponse("200", successApiResponse)));
+
+    return result;
+  }
+
   private static void addExtraParameters(final Components components) {
     components.addParameters(
         "ScrollId" + MODEL_VERSION,
@@ -464,7 +569,7 @@ private static void addExtraParameters(final Components components) {
         "SortBy" + MODEL_VERSION,
         new Parameter()
             .in(NAME_QUERY)
-            .name("sort")
+            .name("sortCriteria")
             .explode(true)
             .description("Sort fields for pagination.")
             .example(PROPERTY_URN)
@@ -472,11 +577,7 @@ private static void addExtraParameters(final Components components) {
                 new Schema()
                     .type(TYPE_ARRAY)
                     ._default(List.of(PROPERTY_URN))
-                    .items(
-                        new Schema<>()
-                            .type(TYPE_STRING)
-                            ._enum(List.of(PROPERTY_URN))
-                            ._default(PROPERTY_URN))));
+                    .items(new Schema<>().type(TYPE_STRING)._default(PROPERTY_URN))));
     components.addParameters(
         "SortOrder" + MODEL_VERSION,
         new Parameter()
@@ -499,7 +600,8 @@ private static void addExtraParameters(final Components components) {
         new Parameter()
             .in(NAME_QUERY)
             .name(NAME_QUERY)
-            .description("Structured search query.")
+            .description(
+                "Structured search query. See Elasticsearch documentation on `query_string` syntax.")
             .example("*")
             .schema(new Schema().type(TYPE_STRING)._default("*")));
   }
@@ -528,6 +630,7 @@ private static Parameter buildParameterSchema(
         .name("aspects")
         .explode(true)
         .description("Aspects to include.")
+        .required(false)
         .example(aspectNames)
         .schema(schema);
   }
@@ -582,6 +685,43 @@ private static void addAspectSchemas(final Components components, final AspectSp
     }
   }
 
+  /**
+   * Generate schema for cross-entity scroll/list response
+   *
+   * @param entityRegistry entity registry
+   * @return schema
+   */
+  private static Schema buildAspectsRefResponseSchema(final EntityRegistry entityRegistry) {
+    final Schema result =
+        new Schema<>()
+            .type(TYPE_OBJECT)
+            .description(ASPECT_DESCRIPTION)
+            .required(List.of(PROPERTY_VALUE));
+
+    entityRegistry
+        .getAspectSpecs()
+        .values()
+        .forEach(
+            aspect ->
+                result.addProperty(
+                    PROPERTY_VALUE, new Schema<>().$ref(PATH_DEFINITIONS + aspect.getName())));
+    result.addProperty(
+        NAME_SYSTEM_METADATA,
+        new Schema<>()
+            .type(TYPE_OBJECT)
+            .anyOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "SystemMetadata")))
+            .description("System metadata for the aspect.")
+            .nullable(true));
+    result.addProperty(
+        NAME_AUDIT_STAMP,
+        new Schema<>()
+            .type(TYPE_OBJECT)
+            .anyOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "AuditStamp")))
+            .description("Audit stamp for the aspect.")
+            .nullable(true));
+    return result;
+  }
+
   private static Schema buildAspectRefResponseSchema(final String aspectName) {
     final Schema result =
         new Schema<>()
@@ -612,7 +752,8 @@ private static Schema buildAspectRefRequestSchema(final String aspectName) {
             .type(TYPE_OBJECT)
             .description(ASPECT_DESCRIPTION)
             .required(List.of(PROPERTY_VALUE))
-            .addProperty(PROPERTY_VALUE, new Schema<>().$ref(PATH_DEFINITIONS + aspectName));
+            .addProperty(
+                PROPERTY_VALUE, new Schema<>().$ref(PATH_DEFINITIONS + toUpperFirst(aspectName)));
     result.addProperty(
         NAME_SYSTEM_METADATA,
         new Schema<>()
@@ -657,6 +798,111 @@ private static Schema buildEntitySchema(
         .properties(properties);
   }
 
+  /**
+   * Generate cross-entity schema
+   *
+   * @param entityRegistry entity registry
+   * @param withSystemMetadata include system metadata
+   * @return schema
+   */
+  private static Schema buildEntitySchema(
+      final EntityRegistry entityRegistry,
+      final Set<String> aspectNames,
+      final boolean withSystemMetadata) {
+    final Map<String, Schema> properties =
+        entityRegistry.getAspectSpecs().entrySet().stream()
+            .filter(a -> aspectNames.contains(a.getValue().getName()))
+            .collect(
+                Collectors.toMap(
+                    Map.Entry::getKey,
+                    a ->
+                        buildAspectRef(
+                            a.getValue().getPegasusSchema().getName(), withSystemMetadata)));
+    properties.put(
+        PROPERTY_URN, new Schema<>().type(TYPE_STRING).description("Unique id for " + ENTITIES));
+
+    return new Schema<>()
+        .type(TYPE_OBJECT)
+        .description(ENTITIES + " object.")
+        .required(List.of(PROPERTY_URN))
+        .properties(properties);
+  }
+
+  /**
+   * Generate cross-entity schema
+   *
+   * @param entityRegistry entity registry
+   * @param definitionNames include aspects
+   * @return schema
+   */
+  private static Schema buildEntitiesRequestSchema(
+      final EntityRegistry entityRegistry, final Set<String> definitionNames) {
+
+    final Set<String> keyAspects = new HashSet<>();
+
+    final List<String> entityNames =
+        entityRegistry.getEntitySpecs().values().stream()
+            .peek(entitySpec -> keyAspects.add(entitySpec.getKeyAspectName()))
+            .map(EntitySpec::getName)
+            .sorted()
+            .toList();
+
+    Schema entitiesSchema =
+        new Schema().type(TYPE_ARRAY).items(new Schema().type(TYPE_STRING)._enum(entityNames));
+
+    final List<String> aspectNames =
+        entityRegistry.getAspectSpecs().values().stream()
+            .filter(aspectSpec -> !aspectSpec.isTimeseries())
+            .map(AspectSpec::getName)
+            .filter(definitionNames::contains) // Only if aspect is defined
+            .distinct()
+            .sorted()
+            .collect(Collectors.toList());
+
+    Schema aspectsSchema =
+        new Schema().type(TYPE_ARRAY).items(new Schema().type(TYPE_STRING)._enum(aspectNames));
+
+    return new Schema<>()
+        .type(TYPE_OBJECT)
+        .description(ENTITIES + " request object.")
+        .example(
+            Map.of(
+                "entities", entityNames.stream().filter(n -> !n.startsWith("dataHub")).toList(),
+                "aspects",
+                    aspectNames.stream()
+                        .filter(n -> !n.startsWith("dataHub") && !keyAspects.contains(n))
+                        .toList()))
+        .properties(
+            Map.of(
+                "entities", entitiesSchema,
+                "aspects", aspectsSchema));
+  }
+
+  /**
+   * Generate schema for cross-entity scroll/list response
+   *
+   * @return schema
+   */
+  private static Schema buildEntitiesScrollSchema() {
+    return new Schema<>()
+        .type(TYPE_OBJECT)
+        .description("Scroll across (list) " + ENTITIES + " objects.")
+        .required(List.of("entities"))
+        .addProperty(
+            NAME_SCROLL_ID,
+            new Schema<>().type(TYPE_STRING).description("Scroll id for pagination."))
+        .addProperty(
+            "entities",
+            new Schema<>()
+                .type(TYPE_ARRAY)
+                .description(ENTITIES + " object.")
+                .items(
+                    new Schema<>()
+                        .$ref(
+                            String.format(
+                                "#/components/schemas/%s%s", ENTITIES, ENTITY_RESPONSE_SUFFIX))));
+  }
+
   private static Schema buildEntityScrollSchema(final EntitySpec entity) {
     return new Schema<>()
         .type(TYPE_OBJECT)
@@ -751,10 +997,10 @@ private static Schema buildAspectPatchSchema() {
   }
 
   private static PathItem buildSingleEntityAspectPath(
-      final EntitySpec entity, final String aspect, final String upperFirstAspect) {
+      final EntitySpec entity, final AspectSpec aspectSpec) {
     final String upperFirstEntity = toUpperFirst(entity.getName());
 
-    List<String> tags = List.of(aspect + " Aspect");
+    List<String> tags = List.of(aspectSpec.getName() + " Aspect");
     // Get Operation
     final Parameter getParameter =
         new Parameter()
@@ -766,7 +1012,10 @@ private static PathItem buildSingleEntityAspectPath(
         new Parameter()
             .in(NAME_QUERY)
             .name(NAME_VERSION)
-            .description("Return a specific aspect version.")
+            .description(
+                aspectSpec.isTimeseries()
+                    ? "This aspect is a `timeseries` aspect, version=0 indicates the most recent aspect should be return. Otherwise return the most recent <= to version as epoch milliseconds."
+                    : "Return a specific aspect version of the aspect.")
             .schema(new Schema().type(TYPE_INTEGER)._default(0).minimum(BigDecimal.ZERO));
     final ApiResponse successApiResponse =
         new ApiResponse()
@@ -781,25 +1030,27 @@ private static PathItem buildSingleEntityAspectPath(
                                     .$ref(
                                         String.format(
                                             "#/components/schemas/%s%s",
-                                            upperFirstAspect, ASPECT_RESPONSE_SUFFIX)))));
+                                            aspectSpec.getPegasusSchema().getName(),
+                                            ASPECT_RESPONSE_SUFFIX)))));
     final Operation getOperation =
         new Operation()
-            .summary(String.format("Get %s for %s.", aspect, entity.getName()))
+            .summary(String.format("Get %s for %s.", aspectSpec.getName(), entity.getName()))
             .tags(tags)
             .parameters(List.of(getParameter, versionParameter))
             .responses(new ApiResponses().addApiResponse("200", successApiResponse));
     // Head Operation
     final ApiResponse successHeadResponse =
         new ApiResponse()
-            .description(String.format("%s on %s exists.", aspect, entity.getName()))
+            .description(String.format("%s on %s exists.", aspectSpec.getName(), entity.getName()))
             .content(new Content().addMediaType("application/json", new MediaType()));
     final ApiResponse notFoundHeadResponse =
         new ApiResponse()
-            .description(String.format("%s on %s does not exist.", aspect, entity.getName()))
+            .description(
+                String.format("%s on %s does not exist.", aspectSpec.getName(), entity.getName()))
             .content(new Content().addMediaType("application/json", new MediaType()));
     final Operation headOperation =
         new Operation()
-            .summary(String.format("%s on %s existence.", aspect, upperFirstEntity))
+            .summary(String.format("%s on %s existence.", aspectSpec.getName(), upperFirstEntity))
             .tags(tags)
             .parameters(
                 List.of(
@@ -815,17 +1066,21 @@ private static PathItem buildSingleEntityAspectPath(
     // Delete Operation
     final ApiResponse successDeleteResponse =
         new ApiResponse()
-            .description(String.format("Delete %s on %s entity.", aspect, upperFirstEntity))
+            .description(
+                String.format("Delete %s on %s entity.", aspectSpec.getName(), upperFirstEntity))
             .content(new Content().addMediaType("application/json", new MediaType()));
     final Operation deleteOperation =
         new Operation()
-            .summary(String.format("Delete %s on entity %s", aspect, upperFirstEntity))
+            .summary(
+                String.format("Delete %s on entity %s", aspectSpec.getName(), upperFirstEntity))
             .tags(tags)
             .responses(new ApiResponses().addApiResponse("200", successDeleteResponse));
     // Post Operation
     final ApiResponse successPostResponse =
         new ApiResponse()
-            .description(String.format("Create aspect %s on %s entity.", aspect, upperFirstEntity))
+            .description(
+                String.format(
+                    "Create aspect %s on %s entity.", aspectSpec.getName(), upperFirstEntity))
             .content(
                 new Content()
                     .addMediaType(
@@ -836,10 +1091,13 @@ private static PathItem buildSingleEntityAspectPath(
                                     .$ref(
                                         String.format(
                                             "#/components/schemas/%s%s",
-                                            upperFirstAspect, ASPECT_RESPONSE_SUFFIX)))));
+                                            aspectSpec.getPegasusSchema().getName(),
+                                            ASPECT_RESPONSE_SUFFIX)))));
     final RequestBody requestBody =
         new RequestBody()
-            .description(String.format("Create aspect %s on %s entity.", aspect, upperFirstEntity))
+            .description(
+                String.format(
+                    "Create aspect %s on %s entity.", aspectSpec.getName(), upperFirstEntity))
             .required(true)
             .content(
                 new Content()
@@ -851,17 +1109,43 @@ private static PathItem buildSingleEntityAspectPath(
                                     .$ref(
                                         String.format(
                                             "#/components/schemas/%s%s",
-                                            upperFirstAspect, ASPECT_REQUEST_SUFFIX)))));
+                                            aspectSpec.getPegasusSchema().getName(),
+                                            ASPECT_REQUEST_SUFFIX)))));
     final Operation postOperation =
         new Operation()
-            .summary(String.format("Create aspect %s on %s ", aspect, upperFirstEntity))
+            .summary(
+                String.format("Create aspect %s on %s ", aspectSpec.getName(), upperFirstEntity))
             .tags(tags)
+            .parameters(
+                List.of(
+                    new Parameter()
+                        .in(NAME_QUERY)
+                        .name("async")
+                        .description("Use async ingestion for high throughput.")
+                        .schema(new Schema().type(TYPE_BOOLEAN)._default(false)),
+                    new Parameter()
+                        .in(NAME_QUERY)
+                        .name(NAME_SYSTEM_METADATA)
+                        .description("Include systemMetadata with response.")
+                        .schema(new Schema().type(TYPE_BOOLEAN)._default(false)),
+                    new Parameter()
+                        .in(NAME_QUERY)
+                        .name("createIfEntityNotExists")
+                        .description("Only create the aspect if the Entity doesn't exist.")
+                        .schema(new Schema().type(TYPE_BOOLEAN)._default(false)),
+                    new Parameter()
+                        .in(NAME_QUERY)
+                        .name("createIfNotExists")
+                        .description("Only create the aspect if the Aspect doesn't exist.")
+                        .schema(new Schema().type(TYPE_BOOLEAN)._default(true))))
             .requestBody(requestBody)
             .responses(new ApiResponses().addApiResponse("201", successPostResponse));
     // Patch Operation
     final ApiResponse successPatchResponse =
         new ApiResponse()
-            .description(String.format("Patch aspect %s on %s entity.", aspect, upperFirstEntity))
+            .description(
+                String.format(
+                    "Patch aspect %s on %s entity.", aspectSpec.getName(), upperFirstEntity))
             .content(
                 new Content()
                     .addMediaType(
@@ -872,10 +1156,13 @@ private static PathItem buildSingleEntityAspectPath(
                                     .$ref(
                                         String.format(
                                             "#/components/schemas/%s%s",
-                                            upperFirstAspect, ASPECT_RESPONSE_SUFFIX)))));
+                                            aspectSpec.getPegasusSchema().getName(),
+                                            ASPECT_RESPONSE_SUFFIX)))));
     final RequestBody patchRequestBody =
         new RequestBody()
-            .description(String.format("Patch aspect %s on %s entity.", aspect, upperFirstEntity))
+            .description(
+                String.format(
+                    "Patch aspect %s on %s entity.", aspectSpec.getName(), upperFirstEntity))
             .required(true)
             .content(
                 new Content()
@@ -892,7 +1179,8 @@ private static PathItem buildSingleEntityAspectPath(
                         .name(NAME_SYSTEM_METADATA)
                         .description("Include systemMetadata with response.")
                         .schema(new Schema().type(TYPE_BOOLEAN)._default(false))))
-            .summary(String.format("Patch aspect %s on %s ", aspect, upperFirstEntity))
+            .summary(
+                String.format("Patch aspect %s on %s ", aspectSpec.getName(), upperFirstEntity))
             .tags(tags)
             .requestBody(patchRequestBody)
             .responses(new ApiResponses().addApiResponse("200", successPatchResponse));
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java
index d7694f3aed933..af13cd3aab051 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java
@@ -13,6 +13,8 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.ByteString;
+import com.linkedin.data.template.SetMode;
+import com.linkedin.data.template.StringMap;
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.AspectRetriever;
@@ -24,11 +26,18 @@
 import com.linkedin.metadata.entity.UpdateAspectResult;
 import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
 import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
+import com.linkedin.metadata.entity.ebean.batch.ProposedItem;
 import com.linkedin.metadata.models.AspectSpec;
+import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.query.filter.SortOrder;
+import com.linkedin.metadata.search.ScrollResult;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchEntityArray;
 import com.linkedin.metadata.utils.AuditStampUtils;
 import com.linkedin.metadata.utils.GenericRecordUtils;
+import com.linkedin.metadata.utils.SearchUtil;
+import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.mxe.SystemMetadata;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.RequestContext;
@@ -37,6 +46,7 @@
 import io.datahubproject.openapi.exception.UnauthorizedException;
 import io.datahubproject.openapi.v3.models.AspectItem;
 import io.datahubproject.openapi.v3.models.GenericAspectV3;
+import io.datahubproject.openapi.v3.models.GenericEntityAspectsBodyV3;
 import io.datahubproject.openapi.v3.models.GenericEntityScrollResultV3;
 import io.datahubproject.openapi.v3.models.GenericEntityV3;
 import io.swagger.v3.oas.annotations.Hidden;
@@ -45,6 +55,9 @@
 import jakarta.servlet.http.HttpServletRequest;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -60,6 +73,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
+import org.springframework.util.CollectionUtils;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -85,7 +99,7 @@ public ResponseEntity<List<GenericEntityV3>> getEntityBatch(
       @RequestBody @Nonnull String jsonEntityList)
       throws URISyntaxException, JsonProcessingException {
 
-    LinkedHashMap<Urn, Map<String, Long>> requestMap = toEntityVersionRequest(jsonEntityList);
+    LinkedHashMap<Urn, Map<AspectSpec, Long>> requestMap = toEntityVersionRequest(jsonEntityList);
 
     Authentication authentication = AuthenticationContext.getAuthentication();
     OperationContext opContext =
@@ -109,7 +123,103 @@ public ResponseEntity<List<GenericEntityV3>> getEntityBatch(
     }
 
     return ResponseEntity.of(
-        Optional.of(buildEntityVersionedAspectList(opContext, requestMap, withSystemMetadata)));
+        Optional.of(
+            buildEntityVersionedAspectList(
+                opContext, requestMap.keySet(), requestMap, withSystemMetadata, true)));
+  }
+
+  @Tag(name = "Generic Entities", description = "API for interacting with generic entities.")
+  @PostMapping(value = "/scroll", produces = MediaType.APPLICATION_JSON_VALUE)
+  @Operation(summary = "Scroll entities")
+  public ResponseEntity<GenericEntityScrollResultV3> scrollEntities(
+      HttpServletRequest request,
+      @RequestParam(value = "count", defaultValue = "10") Integer count,
+      @RequestParam(value = "query", defaultValue = "*") String query,
+      @RequestParam(value = "scrollId", required = false) String scrollId,
+      @RequestParam(value = "sort", required = false, defaultValue = "urn") String sortField,
+      @RequestParam(value = "sortCriteria", required = false) List<String> sortFields,
+      @RequestParam(value = "sortOrder", required = false, defaultValue = "ASCENDING")
+          String sortOrder,
+      @RequestParam(value = "systemMetadata", required = false, defaultValue = "false")
+          Boolean withSystemMetadata,
+      @RequestParam(value = "skipCache", required = false, defaultValue = "false")
+          Boolean skipCache,
+      @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false")
+          Boolean includeSoftDelete,
+      @RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m")
+          String pitKeepALive,
+      @RequestBody @Nonnull GenericEntityAspectsBodyV3 entityAspectsBody)
+      throws URISyntaxException {
+
+    final Collection<String> resolvedEntityNames;
+    if (entityAspectsBody.getEntities() != null) {
+      resolvedEntityNames =
+          entityAspectsBody.getEntities().stream()
+              .map(entityName -> entityRegistry.getEntitySpec(entityName))
+              .map(EntitySpec::getName)
+              .toList();
+    } else {
+      resolvedEntityNames =
+          entityRegistry.getEntitySpecs().values().stream().map(EntitySpec::getName).toList();
+    }
+
+    Authentication authentication = AuthenticationContext.getAuthentication();
+
+    OperationContext opContext =
+        OperationContext.asSession(
+            systemOperationContext,
+            RequestContext.builder()
+                .buildOpenapi(
+                    authentication.getActor().toUrnStr(),
+                    request,
+                    "scrollEntities",
+                    resolvedEntityNames),
+            authorizationChain,
+            authentication,
+            true);
+
+    if (!AuthUtil.isAPIAuthorizedEntityType(opContext, READ, resolvedEntityNames)) {
+      throw new UnauthorizedException(
+          authentication.getActor().toUrnStr() + " is unauthorized to " + READ + "  entities.");
+    }
+
+    List<SortCriterion> sortCriteria;
+    if (!CollectionUtils.isEmpty(sortFields)) {
+      sortCriteria = new ArrayList<>();
+      sortFields.forEach(
+          field -> sortCriteria.add(SearchUtil.sortBy(field, SortOrder.valueOf(sortOrder))));
+    } else {
+      sortCriteria =
+          Collections.singletonList(SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder)));
+    }
+
+    ScrollResult result =
+        searchService.scrollAcrossEntities(
+            opContext
+                .withSearchFlags(flags -> DEFAULT_SEARCH_FLAGS)
+                .withSearchFlags(flags -> flags.setSkipCache(skipCache))
+                .withSearchFlags(flags -> flags.setIncludeSoftDeleted(includeSoftDelete)),
+            resolvedEntityNames,
+            query,
+            null,
+            sortCriteria,
+            scrollId,
+            pitKeepALive,
+            count);
+
+    if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) {
+      throw new UnauthorizedException(
+          authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities.");
+    }
+
+    return ResponseEntity.ok(
+        buildScrollResult(
+            opContext,
+            result.getEntities(),
+            entityAspectsBody.getAspects(),
+            withSystemMetadata,
+            result.getScrollId(),
+            entityAspectsBody.getAspects() != null));
   }
 
   @Override
@@ -118,10 +228,13 @@ public GenericEntityScrollResultV3 buildScrollResult(
       SearchEntityArray searchEntities,
       Set<String> aspectNames,
       boolean withSystemMetadata,
-      @Nullable String scrollId)
+      @Nullable String scrollId,
+      boolean expandEmpty)
       throws URISyntaxException {
     return GenericEntityScrollResultV3.builder()
-        .entities(toRecordTemplates(opContext, searchEntities, aspectNames, withSystemMetadata))
+        .entities(
+            toRecordTemplates(
+                opContext, searchEntities, aspectNames, withSystemMetadata, expandEmpty))
         .scrollId(scrollId)
         .build();
   }
@@ -129,25 +242,65 @@ public GenericEntityScrollResultV3 buildScrollResult(
   @Override
   protected List<GenericEntityV3> buildEntityVersionedAspectList(
       @Nonnull OperationContext opContext,
-      LinkedHashMap<Urn, Map<String, Long>> urnAspectVersions,
-      boolean withSystemMetadata)
+      Collection<Urn> requestedUrns,
+      LinkedHashMap<Urn, Map<AspectSpec, Long>> urnAspectVersions,
+      boolean withSystemMetadata,
+      boolean expandEmpty)
       throws URISyntaxException {
-    if (urnAspectVersions.isEmpty()) {
-      return List.of();
-    } else {
+
+    if (!urnAspectVersions.isEmpty()) {
       Map<Urn, List<EnvelopedAspect>> aspects =
           entityService.getEnvelopedVersionedAspects(
-              opContext, resolveAspectNames(urnAspectVersions, 0L), false);
+              opContext, aspectSpecsToAspectNames(urnAspectVersions, false), false);
+
+      Map<Urn, Map<String, com.linkedin.metadata.aspect.EnvelopedAspect>> timeseriesAspects =
+          aspectSpecsToAspectNames(urnAspectVersions, true).entrySet().stream()
+              .map(
+                  e -> {
+                    // 0 is considered latest due to overlap with versioned and timeseries
+                    Map<String, Long> endTimeMilliMap =
+                        e.getValue().entrySet().stream()
+                            .filter(endTEntry -> endTEntry.getValue() != 0L)
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    return Map.entry(
+                        e.getKey(),
+                        timeseriesAspectService
+                            .getLatestTimeseriesAspectValues(
+                                opContext,
+                                Set.of(e.getKey()),
+                                e.getValue().keySet(),
+                                endTimeMilliMap)
+                            .getOrDefault(e.getKey(), Map.of()));
+                  })
+              .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 
       return urnAspectVersions.keySet().stream()
-          .filter(urn -> aspects.containsKey(urn) && !aspects.get(urn).isEmpty())
+          .filter(
+              urn ->
+                  (aspects.containsKey(urn) && !aspects.get(urn).isEmpty())
+                      || (timeseriesAspects.containsKey(urn)
+                          && !timeseriesAspects.get(urn).isEmpty()))
           .map(
-              u ->
-                  GenericEntityV3.builder()
-                      .build(
-                          objectMapper, u, toAspectItemMap(u, aspects.get(u), withSystemMetadata)))
+              u -> {
+                Map<String, AspectItem> aspectItemMap = new HashMap<>();
+                if (aspects.containsKey(u)) {
+                  aspectItemMap.putAll(toAspectItemMap(u, aspects.get(u), withSystemMetadata));
+                }
+                if (timeseriesAspects.containsKey(u)) {
+                  aspectItemMap.putAll(
+                      toTimeseriesAspectItemMap(u, timeseriesAspects.get(u), withSystemMetadata));
+                }
+
+                return GenericEntityV3.builder().build(objectMapper, u, aspectItemMap);
+              })
+          .collect(Collectors.toList());
+    } else if (!expandEmpty) {
+      return requestedUrns.stream()
+          .map(u -> GenericEntityV3.builder().build(objectMapper, u, Collections.emptyMap()))
           .collect(Collectors.toList());
     }
+
+    return List.of();
   }
 
   private Map<String, AspectItem> toAspectItemMap(
@@ -158,16 +311,36 @@ private Map<String, AspectItem> toAspectItemMap(
                 Map.entry(
                     a.getName(),
                     AspectItem.builder()
-                        .aspect(toRecordTemplate(lookupAspectSpec(urn, a.getName()), a))
+                        .aspect(toRecordTemplate(lookupAspectSpec(urn, a.getName()).get(), a))
                         .systemMetadata(withSystemMetadata ? a.getSystemMetadata() : null)
                         .auditStamp(withSystemMetadata ? a.getCreated() : null)
                         .build()))
         .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
   }
 
+  private Map<String, AspectItem> toTimeseriesAspectItemMap(
+      Urn urn,
+      Map<String, com.linkedin.metadata.aspect.EnvelopedAspect> aspects,
+      boolean withSystemMetadata) {
+    return aspects.entrySet().stream()
+        .map(
+            e ->
+                Map.entry(
+                    e.getKey(),
+                    AspectItem.builder()
+                        .aspect(
+                            toRecordTemplate(lookupAspectSpec(urn, e.getKey()).get(), e.getValue()))
+                        .systemMetadata(
+                            withSystemMetadata ? e.getValue().getSystemMetadata() : null)
+                        .build()))
+        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+  }
+
   @Override
   protected List<GenericEntityV3> buildEntityList(
-      Set<IngestResult> ingestResults, boolean withSystemMetadata) {
+      OperationContext opContext,
+      Collection<IngestResult> ingestResults,
+      boolean withSystemMetadata) {
     List<GenericEntityV3> responseList = new LinkedList<>();
 
     Map<Urn, List<IngestResult>> entityMap =
@@ -214,24 +387,44 @@ protected GenericEntityV3 buildGenericEntity(
                     .build()));
   }
 
+  @Override
+  protected GenericEntityV3 buildGenericEntity(
+      @Nonnull String aspectName, @Nonnull IngestResult ingestResult, boolean withSystemMetadata) {
+    return GenericEntityV3.builder()
+        .build(
+            objectMapper,
+            ingestResult.getUrn(),
+            Map.of(
+                aspectName,
+                AspectItem.builder()
+                    .aspect(ingestResult.getRequest().getRecordTemplate())
+                    .systemMetadata(
+                        withSystemMetadata ? ingestResult.getRequest().getSystemMetadata() : null)
+                    .auditStamp(
+                        withSystemMetadata ? ingestResult.getRequest().getAuditStamp() : null)
+                    .build()));
+  }
+
   private List<GenericEntityV3> toRecordTemplates(
       @Nonnull OperationContext opContext,
       SearchEntityArray searchEntities,
       Set<String> aspectNames,
-      boolean withSystemMetadata)
+      boolean withSystemMetadata,
+      boolean expandEmpty)
       throws URISyntaxException {
     return buildEntityList(
         opContext,
         searchEntities.stream().map(SearchEntity::getEntity).collect(Collectors.toList()),
         aspectNames,
-        withSystemMetadata);
+        withSystemMetadata,
+        expandEmpty);
   }
 
-  private LinkedHashMap<Urn, Map<String, Long>> toEntityVersionRequest(
+  private LinkedHashMap<Urn, Map<AspectSpec, Long>> toEntityVersionRequest(
       @Nonnull String entityArrayList) throws JsonProcessingException, InvalidUrnException {
     JsonNode entities = objectMapper.readTree(entityArrayList);
 
-    LinkedHashMap<Urn, Map<String, Long>> items = new LinkedHashMap<>();
+    LinkedHashMap<Urn, Map<AspectSpec, Long>> items = new LinkedHashMap<>();
     if (entities.isArray()) {
       Iterator<JsonNode> entityItr = entities.iterator();
       while (entityItr.hasNext()) {
@@ -250,7 +443,7 @@ private LinkedHashMap<Urn, Map<String, Long>> toEntityVersionRequest(
             continue;
           }
 
-          AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey());
+          AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()).orElse(null);
 
           if (aspectSpec != null) {
 
@@ -262,10 +455,10 @@ private LinkedHashMap<Urn, Map<String, Long>> toEntityVersionRequest(
               items
                   .get(entityUrn)
                   .put(
-                      aspectSpec.getName(),
+                      aspectSpec,
                       Long.parseLong(headers.getOrDefault(HTTP_HEADER_IF_VERSION_MATCH, "0")));
             } else {
-              items.get(entityUrn).put(aspectSpec.getName(), 0L);
+              items.get(entityUrn).put(aspectSpec, 0L);
             }
           }
         }
@@ -274,7 +467,7 @@ private LinkedHashMap<Urn, Map<String, Long>> toEntityVersionRequest(
         if (items.get(entityUrn).isEmpty()) {
           for (AspectSpec aspectSpec :
               entityRegistry.getEntitySpec(entityUrn.getEntityType()).getAspectSpecs()) {
-            items.get(entityUrn).put(aspectSpec.getName(), 0L);
+            items.get(entityUrn).put(aspectSpec, 0L);
           }
         }
       }
@@ -307,24 +500,45 @@ protected AspectsBatch toMCPBatch(
             continue;
           }
 
-          AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey());
-
-          if (aspectSpec != null) {
-
-            SystemMetadata systemMetadata = null;
-            if (aspect.getValue().has("systemMetadata")) {
-              systemMetadata =
-                  EntityApiUtils.parseSystemMetadata(
-                      objectMapper.writeValueAsString(aspect.getValue().get("systemMetadata")));
-              ((ObjectNode) aspect.getValue()).remove("systemMetadata");
-            }
-            Map<String, String> headers = null;
-            if (aspect.getValue().has("headers")) {
-              headers =
-                  objectMapper.convertValue(
-                      aspect.getValue().get("headers"), new TypeReference<>() {});
-            }
+          AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()).orElse(null);
+          SystemMetadata systemMetadata = null;
+          if (aspect.getValue().has("systemMetadata")) {
+            systemMetadata =
+                EntityApiUtils.parseSystemMetadata(
+                    objectMapper.writeValueAsString(aspect.getValue().get("systemMetadata")));
+            ((ObjectNode) aspect.getValue()).remove("systemMetadata");
+          }
+          Map<String, String> headers = null;
+          if (aspect.getValue().has("headers")) {
+            headers =
+                objectMapper.convertValue(
+                    aspect.getValue().get("headers"), new TypeReference<>() {});
+          }
 
+          JsonNode jsonNodeAspect = aspect.getValue().get("value");
+
+          if (opContext.getValidationContext().isAlternateValidation()) {
+            ProposedItem.ProposedItemBuilder builder =
+                ProposedItem.builder()
+                    .metadataChangeProposal(
+                        new MetadataChangeProposal()
+                            .setEntityUrn(entityUrn)
+                            .setAspectName(aspect.getKey())
+                            .setEntityType(entityUrn.getEntityType())
+                            .setChangeType(ChangeType.UPSERT)
+                            .setAspect(GenericRecordUtils.serializeAspect(jsonNodeAspect))
+                            .setHeaders(
+                                headers != null ? new StringMap(headers) : null,
+                                SetMode.IGNORE_NULL)
+                            .setSystemMetadata(systemMetadata, SetMode.IGNORE_NULL))
+                    .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
+                    .entitySpec(
+                        opContext
+                            .getAspectRetriever()
+                            .getEntityRegistry()
+                            .getEntitySpec(entityUrn.getEntityType()));
+            items.add(builder.build());
+          } else if (aspectSpec != null) {
             ChangeItemImpl.ChangeItemImplBuilder builder =
                 ChangeItemImpl.builder()
                     .urn(entityUrn)
@@ -335,19 +549,19 @@ protected AspectsBatch toMCPBatch(
                     .recordTemplate(
                         GenericRecordUtils.deserializeAspect(
                             ByteString.copyString(
-                                objectMapper.writeValueAsString(aspect.getValue().get("value")),
+                                objectMapper.writeValueAsString(jsonNodeAspect),
                                 StandardCharsets.UTF_8),
                             GenericRecordUtils.JSON,
                             aspectSpec));
 
-            items.add(builder.build(opContext.getRetrieverContext().get().getAspectRetriever()));
+            items.add(builder.build(opContext.getRetrieverContext().getAspectRetriever()));
           }
         }
       }
     }
     return AspectsBatchImpl.builder()
         .items(items)
-        .retrieverContext(opContext.getRetrieverContext().get())
+        .retrieverContext(opContext.getRetrieverContext())
         .build();
   }
 
@@ -356,16 +570,27 @@ protected ChangeMCP toUpsertItem(
       @Nonnull AspectRetriever aspectRetriever,
       Urn entityUrn,
       AspectSpec aspectSpec,
+      Boolean createIfEntityNotExists,
       Boolean createIfNotExists,
       String jsonAspect,
       Actor actor)
       throws JsonProcessingException {
     JsonNode jsonNode = objectMapper.readTree(jsonAspect);
     String aspectJson = jsonNode.get("value").toString();
+
+    final ChangeType changeType;
+    if (Boolean.TRUE.equals(createIfEntityNotExists)) {
+      changeType = ChangeType.CREATE_ENTITY;
+    } else if (Boolean.TRUE.equals(createIfNotExists)) {
+      changeType = ChangeType.CREATE;
+    } else {
+      changeType = ChangeType.UPSERT;
+    }
+
     return ChangeItemImpl.builder()
         .urn(entityUrn)
         .aspectName(aspectSpec.getName())
-        .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT)
+        .changeType(changeType)
         .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
         .recordTemplate(
             GenericRecordUtils.deserializeAspect(
diff --git a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java
index 2dc915f4aaee7..952dc31c5ba38 100644
--- a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java
+++ b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java
@@ -1,6 +1,10 @@
 package io.datahubproject.openapi.v3.controller;
 
 import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME;
+import static com.linkedin.metadata.Constants.DATASET_PROFILE_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
+import static com.linkedin.metadata.utils.GenericRecordUtils.JSON;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyMap;
@@ -9,11 +13,13 @@
 import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.AssertJUnit.assertEquals;
 
 import com.datahub.authentication.Actor;
 import com.datahub.authentication.ActorType;
@@ -21,12 +27,16 @@
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.authorization.AuthorizationResult;
 import com.datahub.authorization.AuthorizerChain;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.linkedin.common.Status;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.template.RecordTemplate;
+import com.linkedin.dataset.DatasetProfile;
 import com.linkedin.entity.Aspect;
 import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.metadata.aspect.batch.AspectsBatch;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.EntityServiceImpl;
 import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService;
@@ -38,13 +48,19 @@
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchEntityArray;
 import com.linkedin.metadata.search.SearchService;
+import com.linkedin.metadata.timeseries.TimeseriesAspectService;
+import com.linkedin.metadata.utils.GenericRecordUtils;
 import com.linkedin.metadata.utils.SearchUtil;
+import com.linkedin.mxe.GenericAspect;
 import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.metadata.context.ValidationContext;
 import io.datahubproject.openapi.config.SpringWebConfig;
+import io.datahubproject.openapi.exception.InvalidUrnException;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
@@ -73,7 +89,9 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests {
   @Autowired private MockMvc mockMvc;
   @Autowired private SearchService mockSearchService;
   @Autowired private EntityService<?> mockEntityService;
+  @Autowired private TimeseriesAspectService mockTimeseriesAspectService;
   @Autowired private EntityRegistry entityRegistry;
+  @Autowired private OperationContext opContext;
 
   @Test
   public void initTest() {
@@ -228,10 +246,149 @@ public void testDeleteEntity() throws Exception {
                         any(), eq(TEST_URN.toString()), eq(aspectName), anyMap(), eq(true)));
   }
 
+  @Test
+  public void testAlternativeMCPValidation() throws InvalidUrnException, JsonProcessingException {
+    final AspectSpec aspectSpec =
+        entityRegistry
+            .getEntitySpec(STRUCTURED_PROPERTY_ENTITY_NAME)
+            .getAspectSpec(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME);
+
+    // Enable Alternative MCP Validation via mock
+    OperationContext opContextSpy = spy(opContext);
+    ValidationContext mockValidationContext = mock(ValidationContext.class);
+    when(mockValidationContext.isAlternateValidation()).thenReturn(true);
+    when(opContextSpy.getValidationContext()).thenReturn(mockValidationContext);
+
+    final String testBody =
+        "[\n"
+            + "    {\n"
+            + "      \"urn\": \"urn:li:structuredProperty:io.acryl.privacy.retentionTime05\",\n"
+            + "      \"propertyDefinition\": {\n"
+            + "        \"value\": {\n"
+            + "          \"allowedValues\": [\n"
+            + "            {\n"
+            + "              \"value\": {\n"
+            + "                \"string\": \"foo2\"\n"
+            + "              },\n"
+            + "              \"description\": \"test foo2 value\"\n"
+            + "            },\n"
+            + "            {\n"
+            + "              \"value\": {\n"
+            + "                \"string\": \"bar2\"\n"
+            + "              },\n"
+            + "              \"description\": \"test bar2 value\"\n"
+            + "            }\n"
+            + "          ],\n"
+            + "          \"entityTypes\": [\n"
+            + "            \"urn:li:entityType:datahub.dataset\"\n"
+            + "          ],\n"
+            + "          \"qualifiedName\": \"io.acryl.privacy.retentionTime05\",\n"
+            + "          \"displayName\": \"Retention Time 03\",\n"
+            + "          \"cardinality\": \"SINGLE\",\n"
+            + "          \"valueType\": \"urn:li:dataType:datahub.string\"\n"
+            + "        }\n"
+            + "      }\n"
+            + "    }\n"
+            + "]";
+
+    AspectsBatch testAspectsBatch =
+        entityController.toMCPBatch(
+            opContextSpy,
+            testBody,
+            opContext.getSessionActorContext().getAuthentication().getActor());
+
+    GenericAspect aspect =
+        testAspectsBatch.getMCPItems().get(0).getMetadataChangeProposal().getAspect();
+    RecordTemplate propertyDefinition =
+        GenericRecordUtils.deserializeAspect(aspect.getValue(), JSON, aspectSpec);
+    assertEquals(
+        propertyDefinition.data().get("entityTypes"), List.of("urn:li:entityType:datahub.dataset"));
+
+    // test alternative
+    reset(mockValidationContext);
+    when(mockValidationContext.isAlternateValidation()).thenReturn(false);
+    testAspectsBatch =
+        entityController.toMCPBatch(
+            opContextSpy,
+            testBody,
+            opContext.getSessionActorContext().getAuthentication().getActor());
+
+    aspect = testAspectsBatch.getMCPItems().get(0).getMetadataChangeProposal().getAspect();
+    propertyDefinition = GenericRecordUtils.deserializeAspect(aspect.getValue(), JSON, aspectSpec);
+    assertEquals(
+        propertyDefinition.data().get("entityTypes"), List.of("urn:li:entityType:datahub.dataset"));
+  }
+
+  @Test
+  public void testTimeseriesAspect() throws Exception {
+    Urn TEST_URN = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:testPlatform,1,PROD)");
+    DatasetProfile firstDatasetProfile =
+        new DatasetProfile()
+            .setRowCount(1)
+            .setColumnCount(10)
+            .setMessageId("testOld")
+            .setTimestampMillis(100);
+    DatasetProfile secondDatasetProfile =
+        new DatasetProfile()
+            .setRowCount(10)
+            .setColumnCount(100)
+            .setMessageId("testLatest")
+            .setTimestampMillis(200);
+
+    // Mock expected timeseries service response
+    when(mockTimeseriesAspectService.getLatestTimeseriesAspectValues(
+            any(OperationContext.class),
+            eq(Set.of(TEST_URN)),
+            eq(Set.of(DATASET_PROFILE_ASPECT_NAME)),
+            eq(Map.of(DATASET_PROFILE_ASPECT_NAME, 150L))))
+        .thenReturn(
+            Map.of(
+                TEST_URN,
+                Map.of(
+                    DATASET_PROFILE_ASPECT_NAME,
+                    new com.linkedin.metadata.aspect.EnvelopedAspect()
+                        .setAspect(GenericRecordUtils.serializeAspect(firstDatasetProfile)))));
+
+    when(mockTimeseriesAspectService.getLatestTimeseriesAspectValues(
+            any(OperationContext.class),
+            eq(Set.of(TEST_URN)),
+            eq(Set.of(DATASET_PROFILE_ASPECT_NAME)),
+            eq(Map.of())))
+        .thenReturn(
+            Map.of(
+                TEST_URN,
+                Map.of(
+                    DATASET_PROFILE_ASPECT_NAME,
+                    new com.linkedin.metadata.aspect.EnvelopedAspect()
+                        .setAspect(GenericRecordUtils.serializeAspect(secondDatasetProfile)))));
+
+    // test timeseries latest aspect
+    mockMvc
+        .perform(
+            MockMvcRequestBuilders.get("/v3/entity/dataset/{urn}/datasetprofile", TEST_URN)
+                .accept(MediaType.APPLICATION_JSON))
+        .andExpect(status().is2xxSuccessful())
+        .andExpect(MockMvcResultMatchers.jsonPath("$.value.rowCount").value(10))
+        .andExpect(MockMvcResultMatchers.jsonPath("$.value.columnCount").value(100))
+        .andExpect(MockMvcResultMatchers.jsonPath("$.value.messageId").value("testLatest"));
+
+    // test oldd aspect
+    mockMvc
+        .perform(
+            MockMvcRequestBuilders.get("/v3/entity/dataset/{urn}/datasetprofile", TEST_URN)
+                .param("version", "150")
+                .accept(MediaType.APPLICATION_JSON))
+        .andExpect(status().is2xxSuccessful())
+        .andExpect(MockMvcResultMatchers.jsonPath("$.value.rowCount").value(1))
+        .andExpect(MockMvcResultMatchers.jsonPath("$.value.columnCount").value(10))
+        .andExpect(MockMvcResultMatchers.jsonPath("$.value.messageId").value("testOld"));
+  }
+
   @TestConfiguration
   public static class EntityControllerTestConfig {
     @MockBean public EntityServiceImpl entityService;
     @MockBean public SearchService searchService;
+    @MockBean public TimeseriesAspectService timeseriesAspectService;
 
     @Bean
     public ObjectMapper objectMapper() {
@@ -268,5 +425,10 @@ public AuthorizerChain authorizerChain() {
 
       return authorizerChain;
     }
+
+    @Bean
+    public TimeseriesAspectService timeseriesAspectService() {
+      return timeseriesAspectService;
+    }
   }
 }
diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java
index b70b643b10f32..4073bff4a22f9 100644
--- a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java
+++ b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java
@@ -72,7 +72,10 @@ public Map<Urn, List<RecordTemplate>> getLatestAspects(
 
   @Override
   public @NotNull Map<String, RecordTemplate> getLatestAspectsForUrn(
-      @Nonnull OperationContext opContext, @Nonnull Urn urn, @Nonnull Set<String> aspectNames) {
+      @Nonnull OperationContext opContext,
+      @Nonnull Urn urn,
+      @Nonnull Set<String> aspectNames,
+      boolean forUpdate) {
     return Collections.emptyMap();
   }
 
diff --git a/metadata-service/plugin/build.gradle b/metadata-service/plugin/build.gradle
index 501589c24d60a..526744ceb85c5 100644
--- a/metadata-service/plugin/build.gradle
+++ b/metadata-service/plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
 
   implementation 'org.apache.commons:commons-lang3:3.12.0'
diff --git a/metadata-service/restli-api/build.gradle b/metadata-service/restli-api/build.gradle
index 505320e8267ee..38a3e9298ff96 100644
--- a/metadata-service/restli-api/build.gradle
+++ b/metadata-service/restli-api/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'pegasus'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   dataModel project(':metadata-models')
   dataModel spec.product.pegasus.restliCommon
diff --git a/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesV2.restspec.json b/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesV2.restspec.json
index 33cfba0f27802..27731af9ffaa7 100644
--- a/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesV2.restspec.json
+++ b/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entitiesV2.restspec.json
@@ -19,6 +19,10 @@
         "name" : "aspects",
         "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
         "optional" : true
+      }, {
+        "name" : "alwaysIncludeKeyAspect",
+        "type" : "boolean",
+        "optional" : true
       } ]
     }, {
       "method" : "batch_get",
@@ -27,6 +31,10 @@
         "name" : "aspects",
         "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
         "optional" : true
+      }, {
+        "name" : "alwaysIncludeKeyAspect",
+        "type" : "boolean",
+        "optional" : true
       } ]
     } ],
     "entity" : {
diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json
index 061feafac1b9b..92abc50abbc4d 100644
--- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json
+++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json
@@ -63,10 +63,10 @@
                           "DESCENDANTS_INCL" : "Represent the relation: URN field any nested children in addition to the given URN",
                           "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
                           "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
-                          "IEQUAL" : "Represent the relation: field = value and support case insensitive values, e.g. platform = hdfs",
                           "EXISTS" : "Represents the relation: field exists and is non-empty, e.g. owners is not null and != [] (empty)",
                           "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
                           "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
+                          "IEQUAL" : "Represent the relation: field = value and support case insensitive values, e.g. platform = hdfs",
                           "IN" : "Represent the relation: String field is one of the array values to, e.g. name in [\"Profile\", \"Event\"]",
                           "IS_NULL" : "Represent the relation: field is null, e.g. platform is null",
                           "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3",
diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
index e9e2778a479d3..1c713fd33884b 100644
--- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
+++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
@@ -169,10 +169,10 @@
                           "DESCENDANTS_INCL" : "Represent the relation: URN field any nested children in addition to the given URN",
                           "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
                           "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
-                          "IEQUAL" : "Represent the relation: field = value and support case insensitive values, e.g. platform = hdfs",
                           "EXISTS" : "Represents the relation: field exists and is non-empty, e.g. owners is not null and != [] (empty)",
                           "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
                           "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
+                          "IEQUAL" : "Represent the relation: field = value and support case insensitive values, e.g. platform = hdfs",
                           "IN" : "Represent the relation: String field is one of the array values to, e.g. name in [\"Profile\", \"Event\"]",
                           "IS_NULL" : "Represent the relation: field is null, e.g. platform is null",
                           "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3",
@@ -941,15 +941,18 @@
             "Searchable" : {
               "/actor" : {
                 "fieldName" : "tagAttributionActors",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/source" : {
                 "fieldName" : "tagAttributionSources",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/time" : {
                 "fieldName" : "tagAttributionDates",
-                "fieldType" : "DATETIME"
+                "fieldType" : "DATETIME",
+                "queryByDefault" : false
               }
             }
           } ]
@@ -1068,15 +1071,18 @@
       "Searchable" : {
         "/actor" : {
           "fieldName" : "termAttributionActors",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/source" : {
           "fieldName" : "termAttributionSources",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/time" : {
           "fieldName" : "termAttributionDates",
-          "fieldType" : "DATETIME"
+          "fieldType" : "DATETIME",
+          "queryByDefault" : false
         }
       }
     } ]
@@ -2570,7 +2576,7 @@
       "Searchable" : {
         "fieldName" : "managerLdap",
         "fieldType" : "URN",
-        "queryByDefault" : true
+        "queryByDefault" : false
       }
     }, {
       "name" : "departmentId",
@@ -2977,7 +2983,7 @@
               "type" : "com.linkedin.dataset.SchemaFieldPath",
               "doc" : "Flattened name of the field. Field is computed from jsonPath field.",
               "Searchable" : {
-                "boostScore" : 5.0,
+                "boostScore" : 1.0,
                 "fieldName" : "fieldPaths",
                 "fieldType" : "TEXT",
                 "queryByDefault" : "true"
@@ -3151,15 +3157,18 @@
               "Searchable" : {
                 "/tags/*/attribution/actor" : {
                   "fieldName" : "fieldTagAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/source" : {
                   "fieldName" : "fieldTagAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/time" : {
                   "fieldName" : "fieldTagAttributionDates",
-                  "fieldType" : "DATETIME"
+                  "fieldType" : "DATETIME",
+                  "queryByDefault" : false
                 },
                 "/tags/*/tag" : {
                   "boostScore" : 0.5,
@@ -3181,15 +3190,18 @@
               "Searchable" : {
                 "/terms/*/attribution/actor" : {
                   "fieldName" : "fieldTermAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/source" : {
                   "fieldName" : "fieldTermAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/time" : {
                   "fieldName" : "fieldTermAttributionDates",
-                  "fieldType" : "DATETIME"
+                  "fieldType" : "DATETIME",
+                  "queryByDefault" : false
                 },
                 "/terms/*/urn" : {
                   "boostScore" : 0.5,
@@ -3362,11 +3374,13 @@
               "Searchable" : {
                 "/tags/*/attribution/actor" : {
                   "fieldName" : "editedFieldTagAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/source" : {
                   "fieldName" : "editedFieldTagAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/time" : {
                   "fieldName" : "editedFieldTagAttributionDates",
@@ -3392,11 +3406,13 @@
               "Searchable" : {
                 "/terms/*/attribution/actor" : {
                   "fieldName" : "editedFieldTermAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/source" : {
                   "fieldName" : "editedFieldTermAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/time" : {
                   "fieldName" : "editedFieldTermAttributionDates",
@@ -3810,12 +3826,23 @@
       "type" : "record",
       "name" : "MLModelProperties",
       "namespace" : "com.linkedin.ml.metadata",
-      "doc" : "Properties associated with a ML Model",
+      "doc" : "Properties associated with a ML Model\r",
       "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ],
       "fields" : [ {
+        "name" : "name",
+        "type" : "string",
+        "doc" : "Display name of the MLModel\r",
+        "optional" : true,
+        "Searchable" : {
+          "boostScore" : 10.0,
+          "enableAutocomplete" : true,
+          "fieldType" : "WORD_GRAM",
+          "queryByDefault" : true
+        }
+      }, {
         "name" : "description",
         "type" : "string",
-        "doc" : "Documentation of the MLModel",
+        "doc" : "Documentation of the MLModel\r",
         "optional" : true,
         "Searchable" : {
           "fieldType" : "TEXT",
@@ -3824,17 +3851,28 @@
       }, {
         "name" : "date",
         "type" : "com.linkedin.common.Time",
-        "doc" : "Date when the MLModel was developed",
+        "doc" : "Date when the MLModel was developed\r",
+        "optional" : true,
+        "deprecated" : true
+      }, {
+        "name" : "created",
+        "type" : "com.linkedin.common.TimeStamp",
+        "doc" : "Audit stamp containing who created this and when\r",
+        "optional" : true
+      }, {
+        "name" : "lastModified",
+        "type" : "com.linkedin.common.TimeStamp",
+        "doc" : "Date when the MLModel was last modified\r",
         "optional" : true
       }, {
         "name" : "version",
         "type" : "com.linkedin.common.VersionTag",
-        "doc" : "Version of the MLModel",
+        "doc" : "Version of the MLModel\r",
         "optional" : true
       }, {
         "name" : "type",
         "type" : "string",
-        "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc",
+        "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r",
         "optional" : true,
         "Searchable" : {
           "fieldType" : "TEXT_PARTIAL"
@@ -3850,7 +3888,7 @@
             "ref" : [ "string", "int", "float", "double", "boolean" ]
           }
         },
-        "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams",
+        "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r",
         "optional" : true
       }, {
         "name" : "hyperParams",
@@ -3885,7 +3923,7 @@
             }
           }
         },
-        "doc" : "Hyperparameters of the MLModel",
+        "doc" : "Hyperparameters of the MLModel\r",
         "optional" : true
       }, {
         "name" : "trainingMetrics",
@@ -3920,7 +3958,7 @@
             }
           }
         },
-        "doc" : "Metrics of the MLModel used in training",
+        "doc" : "Metrics of the MLModel used in training\r",
         "optional" : true
       }, {
         "name" : "onlineMetrics",
@@ -3928,7 +3966,7 @@
           "type" : "array",
           "items" : "MLMetric"
         },
-        "doc" : "Metrics of the MLModel used in production",
+        "doc" : "Metrics of the MLModel used in production\r",
         "optional" : true
       }, {
         "name" : "mlFeatures",
@@ -3936,7 +3974,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.MLFeatureUrn"
         },
-        "doc" : "List of features used for MLModel training",
+        "doc" : "List of features used for MLModel training\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -3951,7 +3989,7 @@
           "type" : "array",
           "items" : "string"
         },
-        "doc" : "Tags for the MLModel",
+        "doc" : "Tags for the MLModel\r",
         "default" : [ ]
       }, {
         "name" : "deployments",
@@ -3959,7 +3997,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "Deployments for the MLModel",
+        "doc" : "Deployments for the MLModel\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -3973,11 +4011,11 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "List of jobs (if any) used to train the model",
+        "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
-            "entityTypes" : [ "dataJob" ],
+            "entityTypes" : [ "dataJob", "dataProcessInstance" ],
             "isLineage" : true,
             "name" : "TrainedBy"
           }
@@ -3988,7 +4026,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "List of jobs (if any) that use the model",
+        "doc" : "List of jobs (if any) that use the model\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -4004,7 +4042,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "Groups the model belongs to",
+        "doc" : "Groups the model belongs to\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
index 959cb5381fd9b..77d4644f3c121 100644
--- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
+++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
@@ -932,15 +932,18 @@
             "Searchable" : {
               "/actor" : {
                 "fieldName" : "tagAttributionActors",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/source" : {
                 "fieldName" : "tagAttributionSources",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/time" : {
                 "fieldName" : "tagAttributionDates",
-                "fieldType" : "DATETIME"
+                "fieldType" : "DATETIME",
+                "queryByDefault" : false
               }
             }
           } ]
@@ -1059,15 +1062,18 @@
       "Searchable" : {
         "/actor" : {
           "fieldName" : "termAttributionActors",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/source" : {
           "fieldName" : "termAttributionSources",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/time" : {
           "fieldName" : "termAttributionDates",
-          "fieldType" : "DATETIME"
+          "fieldType" : "DATETIME",
+          "queryByDefault" : false
         }
       }
     } ]
@@ -2783,7 +2789,7 @@
                     "Searchable" : {
                       "fieldName" : "managerLdap",
                       "fieldType" : "URN",
-                      "queryByDefault" : true
+                      "queryByDefault" : false
                     }
                   }, {
                     "name" : "departmentId",
@@ -3365,7 +3371,7 @@
                           "type" : "com.linkedin.dataset.SchemaFieldPath",
                           "doc" : "Flattened name of the field. Field is computed from jsonPath field.",
                           "Searchable" : {
-                            "boostScore" : 5.0,
+                            "boostScore" : 1.0,
                             "fieldName" : "fieldPaths",
                             "fieldType" : "TEXT",
                             "queryByDefault" : "true"
@@ -3539,15 +3545,18 @@
                           "Searchable" : {
                             "/tags/*/attribution/actor" : {
                               "fieldName" : "fieldTagAttributionActors",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/tags/*/attribution/source" : {
                               "fieldName" : "fieldTagAttributionSources",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/tags/*/attribution/time" : {
                               "fieldName" : "fieldTagAttributionDates",
-                              "fieldType" : "DATETIME"
+                              "fieldType" : "DATETIME",
+                              "queryByDefault" : false
                             },
                             "/tags/*/tag" : {
                               "boostScore" : 0.5,
@@ -3569,15 +3578,18 @@
                           "Searchable" : {
                             "/terms/*/attribution/actor" : {
                               "fieldName" : "fieldTermAttributionActors",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/terms/*/attribution/source" : {
                               "fieldName" : "fieldTermAttributionSources",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/terms/*/attribution/time" : {
                               "fieldName" : "fieldTermAttributionDates",
-                              "fieldType" : "DATETIME"
+                              "fieldType" : "DATETIME",
+                              "queryByDefault" : false
                             },
                             "/terms/*/urn" : {
                               "boostScore" : 0.5,
@@ -3750,11 +3762,13 @@
                           "Searchable" : {
                             "/tags/*/attribution/actor" : {
                               "fieldName" : "editedFieldTagAttributionActors",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/tags/*/attribution/source" : {
                               "fieldName" : "editedFieldTagAttributionSources",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/tags/*/attribution/time" : {
                               "fieldName" : "editedFieldTagAttributionDates",
@@ -3780,11 +3794,13 @@
                           "Searchable" : {
                             "/terms/*/attribution/actor" : {
                               "fieldName" : "editedFieldTermAttributionActors",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/terms/*/attribution/source" : {
                               "fieldName" : "editedFieldTermAttributionSources",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/terms/*/attribution/time" : {
                               "fieldName" : "editedFieldTermAttributionDates",
@@ -3968,12 +3984,23 @@
                   "type" : "record",
                   "name" : "MLModelProperties",
                   "namespace" : "com.linkedin.ml.metadata",
-                  "doc" : "Properties associated with a ML Model",
+                  "doc" : "Properties associated with a ML Model\r",
                   "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ],
                   "fields" : [ {
+                    "name" : "name",
+                    "type" : "string",
+                    "doc" : "Display name of the MLModel\r",
+                    "optional" : true,
+                    "Searchable" : {
+                      "boostScore" : 10.0,
+                      "enableAutocomplete" : true,
+                      "fieldType" : "WORD_GRAM",
+                      "queryByDefault" : true
+                    }
+                  }, {
                     "name" : "description",
                     "type" : "string",
-                    "doc" : "Documentation of the MLModel",
+                    "doc" : "Documentation of the MLModel\r",
                     "optional" : true,
                     "Searchable" : {
                       "fieldType" : "TEXT",
@@ -3982,17 +4009,28 @@
                   }, {
                     "name" : "date",
                     "type" : "com.linkedin.common.Time",
-                    "doc" : "Date when the MLModel was developed",
+                    "doc" : "Date when the MLModel was developed\r",
+                    "optional" : true,
+                    "deprecated" : true
+                  }, {
+                    "name" : "created",
+                    "type" : "com.linkedin.common.TimeStamp",
+                    "doc" : "Audit stamp containing who created this and when\r",
+                    "optional" : true
+                  }, {
+                    "name" : "lastModified",
+                    "type" : "com.linkedin.common.TimeStamp",
+                    "doc" : "Date when the MLModel was last modified\r",
                     "optional" : true
                   }, {
                     "name" : "version",
                     "type" : "com.linkedin.common.VersionTag",
-                    "doc" : "Version of the MLModel",
+                    "doc" : "Version of the MLModel\r",
                     "optional" : true
                   }, {
                     "name" : "type",
                     "type" : "string",
-                    "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc",
+                    "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r",
                     "optional" : true,
                     "Searchable" : {
                       "fieldType" : "TEXT_PARTIAL"
@@ -4008,7 +4046,7 @@
                         "ref" : [ "string", "int", "float", "double", "boolean" ]
                       }
                     },
-                    "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams",
+                    "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r",
                     "optional" : true
                   }, {
                     "name" : "hyperParams",
@@ -4043,7 +4081,7 @@
                         }
                       }
                     },
-                    "doc" : "Hyperparameters of the MLModel",
+                    "doc" : "Hyperparameters of the MLModel\r",
                     "optional" : true
                   }, {
                     "name" : "trainingMetrics",
@@ -4078,7 +4116,7 @@
                         }
                       }
                     },
-                    "doc" : "Metrics of the MLModel used in training",
+                    "doc" : "Metrics of the MLModel used in training\r",
                     "optional" : true
                   }, {
                     "name" : "onlineMetrics",
@@ -4086,7 +4124,7 @@
                       "type" : "array",
                       "items" : "MLMetric"
                     },
-                    "doc" : "Metrics of the MLModel used in production",
+                    "doc" : "Metrics of the MLModel used in production\r",
                     "optional" : true
                   }, {
                     "name" : "mlFeatures",
@@ -4094,7 +4132,7 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.MLFeatureUrn"
                     },
-                    "doc" : "List of features used for MLModel training",
+                    "doc" : "List of features used for MLModel training\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
@@ -4109,7 +4147,7 @@
                       "type" : "array",
                       "items" : "string"
                     },
-                    "doc" : "Tags for the MLModel",
+                    "doc" : "Tags for the MLModel\r",
                     "default" : [ ]
                   }, {
                     "name" : "deployments",
@@ -4117,7 +4155,7 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.Urn"
                     },
-                    "doc" : "Deployments for the MLModel",
+                    "doc" : "Deployments for the MLModel\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
@@ -4131,11 +4169,11 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.Urn"
                     },
-                    "doc" : "List of jobs (if any) used to train the model",
+                    "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
-                        "entityTypes" : [ "dataJob" ],
+                        "entityTypes" : [ "dataJob", "dataProcessInstance" ],
                         "isLineage" : true,
                         "name" : "TrainedBy"
                       }
@@ -4146,7 +4184,7 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.Urn"
                     },
-                    "doc" : "List of jobs (if any) that use the model",
+                    "doc" : "List of jobs (if any) that use the model\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
@@ -4162,7 +4200,7 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.Urn"
                     },
-                    "doc" : "Groups the model belongs to",
+                    "doc" : "Groups the model belongs to\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
@@ -4965,12 +5003,23 @@
                   "type" : "record",
                   "name" : "MLModelGroupProperties",
                   "namespace" : "com.linkedin.ml.metadata",
-                  "doc" : "Properties associated with an ML Model Group",
+                  "doc" : "Properties associated with an ML Model Group\r",
                   "include" : [ "com.linkedin.common.CustomProperties" ],
                   "fields" : [ {
+                    "name" : "name",
+                    "type" : "string",
+                    "doc" : "Display name of the MLModelGroup\r",
+                    "optional" : true,
+                    "Searchable" : {
+                      "boostScore" : 10.0,
+                      "enableAutocomplete" : true,
+                      "fieldType" : "WORD_GRAM",
+                      "queryByDefault" : true
+                    }
+                  }, {
                     "name" : "description",
                     "type" : "string",
-                    "doc" : "Documentation of the MLModelGroup",
+                    "doc" : "Documentation of the MLModelGroup\r",
                     "optional" : true,
                     "Searchable" : {
                       "fieldType" : "TEXT",
@@ -4979,12 +5028,38 @@
                   }, {
                     "name" : "createdAt",
                     "type" : "com.linkedin.common.Time",
-                    "doc" : "Date when the MLModelGroup was developed",
+                    "doc" : "Date when the MLModelGroup was developed\r",
+                    "optional" : true,
+                    "deprecated" : true
+                  }, {
+                    "name" : "created",
+                    "type" : "com.linkedin.common.TimeStamp",
+                    "doc" : "Time and Actor who created the MLModelGroup\r",
                     "optional" : true
+                  }, {
+                    "name" : "lastModified",
+                    "type" : "com.linkedin.common.TimeStamp",
+                    "doc" : "Date when the MLModelGroup was last modified\r",
+                    "optional" : true
+                  }, {
+                    "name" : "trainingJobs",
+                    "type" : {
+                      "type" : "array",
+                      "items" : "com.linkedin.common.Urn"
+                    },
+                    "doc" : "List of jobs (if any) used to train the model group. Visible in Lineage.\r",
+                    "optional" : true,
+                    "Relationship" : {
+                      "/*" : {
+                        "entityTypes" : [ "dataJob" ],
+                        "isLineage" : true,
+                        "name" : "TrainedBy"
+                      }
+                    }
                   }, {
                     "name" : "version",
                     "type" : "com.linkedin.common.VersionTag",
-                    "doc" : "Version of the MLModelGroup",
+                    "doc" : "Version of the MLModelGroup\r",
                     "optional" : true
                   } ],
                   "Aspect" : {
@@ -5204,9 +5279,9 @@
                     },
                     "Searchable" : {
                       "/*" : {
-                        "boostScore" : 2.0,
                         "fieldName" : "isRelatedTerms",
-                        "fieldType" : "URN"
+                        "fieldType" : "URN",
+                        "queryByDefault" : false
                       }
                     }
                   }, {
@@ -5225,9 +5300,9 @@
                     },
                     "Searchable" : {
                       "/*" : {
-                        "boostScore" : 2.0,
                         "fieldName" : "hasRelatedTerms",
-                        "fieldType" : "URN"
+                        "fieldType" : "URN",
+                        "queryByDefault" : false
                       }
                     }
                   }, {
@@ -6068,23 +6143,29 @@
       "doc" : "invoke query rewrite chain for filters based on configured rewriters",
       "default" : true,
       "optional" : true
+    }, {
+      "name" : "includeDefaultFacets",
+      "type" : "boolean",
+      "doc" : "Include default facets when getting facets to aggregate on in search requests.\nBy default we include these, but custom aggregation requests don't need them.",
+      "default" : true,
+      "optional" : true
     } ]
   }, {
     "type" : "enum",
     "name" : "Condition",
     "namespace" : "com.linkedin.metadata.query.filter",
     "doc" : "The matching condition in a filter criterion",
-    "symbols" : [ "CONTAIN", "END_WITH", "EQUAL","IEQUAL", "IS_NULL", "EXISTS", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH", "DESCENDANTS_INCL", "ANCESTORS_INCL", "RELATED_INCL" ],
+    "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "IEQUAL", "IS_NULL", "EXISTS", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH", "DESCENDANTS_INCL", "ANCESTORS_INCL", "RELATED_INCL" ],
     "symbolDocs" : {
       "ANCESTORS_INCL" : "Represent the relation: URN field matches any nested parent in addition to the given URN",
       "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile",
       "DESCENDANTS_INCL" : "Represent the relation: URN field any nested children in addition to the given URN",
       "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
       "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
-      "IEQUAL" : "Represent the relation: field = value and support case insensitive values, e.g. platform = hdfs",
       "EXISTS" : "Represents the relation: field exists and is non-empty, e.g. owners is not null and != [] (empty)",
       "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
       "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
+      "IEQUAL" : "Represent the relation: field = value and support case insensitive values, e.g. platform = hdfs",
       "IN" : "Represent the relation: String field is one of the array values to, e.g. name in [\"Profile\", \"Event\"]",
       "IS_NULL" : "Represent the relation: field is null, e.g. platform is null",
       "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3",
@@ -6314,6 +6395,11 @@
       "type" : "string",
       "doc" : "Name of the filter to be displayed in the UI",
       "optional" : true
+    }, {
+      "name" : "entity",
+      "type" : "com.linkedin.common.Urn",
+      "doc" : "Entity associated with this facet",
+      "optional" : true
     }, {
       "name" : "aggregations",
       "type" : {
diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json
index 9bf7f97b34be1..9c5f41281fcfb 100644
--- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json
+++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json
@@ -182,6 +182,10 @@
           "name" : "aspects",
           "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
           "optional" : true
+        }, {
+          "name" : "alwaysIncludeKeyAspect",
+          "type" : "boolean",
+          "optional" : true
         } ]
       }, {
         "method" : "batch_get",
@@ -190,6 +194,10 @@
           "name" : "aspects",
           "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
           "optional" : true
+        }, {
+          "name" : "alwaysIncludeKeyAspect",
+          "type" : "boolean",
+          "optional" : true
         } ]
       } ],
       "entity" : {
diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json
index 3e0cd46aba0c0..8b6def75f7a66 100644
--- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json
+++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json
@@ -674,15 +674,18 @@
             "Searchable" : {
               "/actor" : {
                 "fieldName" : "tagAttributionActors",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/source" : {
                 "fieldName" : "tagAttributionSources",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/time" : {
                 "fieldName" : "tagAttributionDates",
-                "fieldType" : "DATETIME"
+                "fieldType" : "DATETIME",
+                "queryByDefault" : false
               }
             }
           } ]
@@ -801,15 +804,18 @@
       "Searchable" : {
         "/actor" : {
           "fieldName" : "termAttributionActors",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/source" : {
           "fieldName" : "termAttributionSources",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/time" : {
           "fieldName" : "termAttributionDates",
-          "fieldType" : "DATETIME"
+          "fieldType" : "DATETIME",
+          "queryByDefault" : false
         }
       }
     } ]
@@ -2294,7 +2300,7 @@
       "Searchable" : {
         "fieldName" : "managerLdap",
         "fieldType" : "URN",
-        "queryByDefault" : true
+        "queryByDefault" : false
       }
     }, {
       "name" : "departmentId",
@@ -2701,7 +2707,7 @@
               "type" : "com.linkedin.dataset.SchemaFieldPath",
               "doc" : "Flattened name of the field. Field is computed from jsonPath field.",
               "Searchable" : {
-                "boostScore" : 5.0,
+                "boostScore" : 1.0,
                 "fieldName" : "fieldPaths",
                 "fieldType" : "TEXT",
                 "queryByDefault" : "true"
@@ -2875,15 +2881,18 @@
               "Searchable" : {
                 "/tags/*/attribution/actor" : {
                   "fieldName" : "fieldTagAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/source" : {
                   "fieldName" : "fieldTagAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/time" : {
                   "fieldName" : "fieldTagAttributionDates",
-                  "fieldType" : "DATETIME"
+                  "fieldType" : "DATETIME",
+                  "queryByDefault" : false
                 },
                 "/tags/*/tag" : {
                   "boostScore" : 0.5,
@@ -2905,15 +2914,18 @@
               "Searchable" : {
                 "/terms/*/attribution/actor" : {
                   "fieldName" : "fieldTermAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/source" : {
                   "fieldName" : "fieldTermAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/time" : {
                   "fieldName" : "fieldTermAttributionDates",
-                  "fieldType" : "DATETIME"
+                  "fieldType" : "DATETIME",
+                  "queryByDefault" : false
                 },
                 "/terms/*/urn" : {
                   "boostScore" : 0.5,
@@ -3086,11 +3098,13 @@
               "Searchable" : {
                 "/tags/*/attribution/actor" : {
                   "fieldName" : "editedFieldTagAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/source" : {
                   "fieldName" : "editedFieldTagAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/time" : {
                   "fieldName" : "editedFieldTagAttributionDates",
@@ -3116,11 +3130,13 @@
               "Searchable" : {
                 "/terms/*/attribution/actor" : {
                   "fieldName" : "editedFieldTermAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/source" : {
                   "fieldName" : "editedFieldTermAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/time" : {
                   "fieldName" : "editedFieldTermAttributionDates",
@@ -3534,12 +3550,23 @@
       "type" : "record",
       "name" : "MLModelProperties",
       "namespace" : "com.linkedin.ml.metadata",
-      "doc" : "Properties associated with a ML Model",
+      "doc" : "Properties associated with a ML Model\r",
       "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ],
       "fields" : [ {
+        "name" : "name",
+        "type" : "string",
+        "doc" : "Display name of the MLModel\r",
+        "optional" : true,
+        "Searchable" : {
+          "boostScore" : 10.0,
+          "enableAutocomplete" : true,
+          "fieldType" : "WORD_GRAM",
+          "queryByDefault" : true
+        }
+      }, {
         "name" : "description",
         "type" : "string",
-        "doc" : "Documentation of the MLModel",
+        "doc" : "Documentation of the MLModel\r",
         "optional" : true,
         "Searchable" : {
           "fieldType" : "TEXT",
@@ -3548,17 +3575,28 @@
       }, {
         "name" : "date",
         "type" : "com.linkedin.common.Time",
-        "doc" : "Date when the MLModel was developed",
+        "doc" : "Date when the MLModel was developed\r",
+        "optional" : true,
+        "deprecated" : true
+      }, {
+        "name" : "created",
+        "type" : "com.linkedin.common.TimeStamp",
+        "doc" : "Audit stamp containing who created this and when\r",
+        "optional" : true
+      }, {
+        "name" : "lastModified",
+        "type" : "com.linkedin.common.TimeStamp",
+        "doc" : "Date when the MLModel was last modified\r",
         "optional" : true
       }, {
         "name" : "version",
         "type" : "com.linkedin.common.VersionTag",
-        "doc" : "Version of the MLModel",
+        "doc" : "Version of the MLModel\r",
         "optional" : true
       }, {
         "name" : "type",
         "type" : "string",
-        "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc",
+        "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r",
         "optional" : true,
         "Searchable" : {
           "fieldType" : "TEXT_PARTIAL"
@@ -3574,7 +3612,7 @@
             "ref" : [ "string", "int", "float", "double", "boolean" ]
           }
         },
-        "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams",
+        "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r",
         "optional" : true
       }, {
         "name" : "hyperParams",
@@ -3609,7 +3647,7 @@
             }
           }
         },
-        "doc" : "Hyperparameters of the MLModel",
+        "doc" : "Hyperparameters of the MLModel\r",
         "optional" : true
       }, {
         "name" : "trainingMetrics",
@@ -3644,7 +3682,7 @@
             }
           }
         },
-        "doc" : "Metrics of the MLModel used in training",
+        "doc" : "Metrics of the MLModel used in training\r",
         "optional" : true
       }, {
         "name" : "onlineMetrics",
@@ -3652,7 +3690,7 @@
           "type" : "array",
           "items" : "MLMetric"
         },
-        "doc" : "Metrics of the MLModel used in production",
+        "doc" : "Metrics of the MLModel used in production\r",
         "optional" : true
       }, {
         "name" : "mlFeatures",
@@ -3660,7 +3698,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.MLFeatureUrn"
         },
-        "doc" : "List of features used for MLModel training",
+        "doc" : "List of features used for MLModel training\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -3675,7 +3713,7 @@
           "type" : "array",
           "items" : "string"
         },
-        "doc" : "Tags for the MLModel",
+        "doc" : "Tags for the MLModel\r",
         "default" : [ ]
       }, {
         "name" : "deployments",
@@ -3683,7 +3721,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "Deployments for the MLModel",
+        "doc" : "Deployments for the MLModel\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -3697,11 +3735,11 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "List of jobs (if any) used to train the model",
+        "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
-            "entityTypes" : [ "dataJob" ],
+            "entityTypes" : [ "dataJob", "dataProcessInstance" ],
             "isLineage" : true,
             "name" : "TrainedBy"
           }
@@ -3712,7 +3750,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "List of jobs (if any) that use the model",
+        "doc" : "List of jobs (if any) that use the model\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -3728,7 +3766,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "Groups the model belongs to",
+        "doc" : "Groups the model belongs to\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json
index 7f651a10139e2..e4cc5c42303ee 100644
--- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json
+++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json
@@ -674,15 +674,18 @@
             "Searchable" : {
               "/actor" : {
                 "fieldName" : "tagAttributionActors",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/source" : {
                 "fieldName" : "tagAttributionSources",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/time" : {
                 "fieldName" : "tagAttributionDates",
-                "fieldType" : "DATETIME"
+                "fieldType" : "DATETIME",
+                "queryByDefault" : false
               }
             }
           } ]
@@ -801,15 +804,18 @@
       "Searchable" : {
         "/actor" : {
           "fieldName" : "termAttributionActors",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/source" : {
           "fieldName" : "termAttributionSources",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/time" : {
           "fieldName" : "termAttributionDates",
-          "fieldType" : "DATETIME"
+          "fieldType" : "DATETIME",
+          "queryByDefault" : false
         }
       }
     } ]
@@ -2288,7 +2294,7 @@
       "Searchable" : {
         "fieldName" : "managerLdap",
         "fieldType" : "URN",
-        "queryByDefault" : true
+        "queryByDefault" : false
       }
     }, {
       "name" : "departmentId",
@@ -2695,7 +2701,7 @@
               "type" : "com.linkedin.dataset.SchemaFieldPath",
               "doc" : "Flattened name of the field. Field is computed from jsonPath field.",
               "Searchable" : {
-                "boostScore" : 5.0,
+                "boostScore" : 1.0,
                 "fieldName" : "fieldPaths",
                 "fieldType" : "TEXT",
                 "queryByDefault" : "true"
@@ -2869,15 +2875,18 @@
               "Searchable" : {
                 "/tags/*/attribution/actor" : {
                   "fieldName" : "fieldTagAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/source" : {
                   "fieldName" : "fieldTagAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/time" : {
                   "fieldName" : "fieldTagAttributionDates",
-                  "fieldType" : "DATETIME"
+                  "fieldType" : "DATETIME",
+                  "queryByDefault" : false
                 },
                 "/tags/*/tag" : {
                   "boostScore" : 0.5,
@@ -2899,15 +2908,18 @@
               "Searchable" : {
                 "/terms/*/attribution/actor" : {
                   "fieldName" : "fieldTermAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/source" : {
                   "fieldName" : "fieldTermAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/time" : {
                   "fieldName" : "fieldTermAttributionDates",
-                  "fieldType" : "DATETIME"
+                  "fieldType" : "DATETIME",
+                  "queryByDefault" : false
                 },
                 "/terms/*/urn" : {
                   "boostScore" : 0.5,
@@ -3080,11 +3092,13 @@
               "Searchable" : {
                 "/tags/*/attribution/actor" : {
                   "fieldName" : "editedFieldTagAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/source" : {
                   "fieldName" : "editedFieldTagAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/tags/*/attribution/time" : {
                   "fieldName" : "editedFieldTagAttributionDates",
@@ -3110,11 +3124,13 @@
               "Searchable" : {
                 "/terms/*/attribution/actor" : {
                   "fieldName" : "editedFieldTermAttributionActors",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/source" : {
                   "fieldName" : "editedFieldTermAttributionSources",
-                  "fieldType" : "URN"
+                  "fieldType" : "URN",
+                  "queryByDefault" : false
                 },
                 "/terms/*/attribution/time" : {
                   "fieldName" : "editedFieldTermAttributionDates",
@@ -3528,12 +3544,23 @@
       "type" : "record",
       "name" : "MLModelProperties",
       "namespace" : "com.linkedin.ml.metadata",
-      "doc" : "Properties associated with a ML Model",
+      "doc" : "Properties associated with a ML Model\r",
       "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ],
       "fields" : [ {
+        "name" : "name",
+        "type" : "string",
+        "doc" : "Display name of the MLModel\r",
+        "optional" : true,
+        "Searchable" : {
+          "boostScore" : 10.0,
+          "enableAutocomplete" : true,
+          "fieldType" : "WORD_GRAM",
+          "queryByDefault" : true
+        }
+      }, {
         "name" : "description",
         "type" : "string",
-        "doc" : "Documentation of the MLModel",
+        "doc" : "Documentation of the MLModel\r",
         "optional" : true,
         "Searchable" : {
           "fieldType" : "TEXT",
@@ -3542,17 +3569,28 @@
       }, {
         "name" : "date",
         "type" : "com.linkedin.common.Time",
-        "doc" : "Date when the MLModel was developed",
+        "doc" : "Date when the MLModel was developed\r",
+        "optional" : true,
+        "deprecated" : true
+      }, {
+        "name" : "created",
+        "type" : "com.linkedin.common.TimeStamp",
+        "doc" : "Audit stamp containing who created this and when\r",
+        "optional" : true
+      }, {
+        "name" : "lastModified",
+        "type" : "com.linkedin.common.TimeStamp",
+        "doc" : "Date when the MLModel was last modified\r",
         "optional" : true
       }, {
         "name" : "version",
         "type" : "com.linkedin.common.VersionTag",
-        "doc" : "Version of the MLModel",
+        "doc" : "Version of the MLModel\r",
         "optional" : true
       }, {
         "name" : "type",
         "type" : "string",
-        "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc",
+        "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r",
         "optional" : true,
         "Searchable" : {
           "fieldType" : "TEXT_PARTIAL"
@@ -3568,7 +3606,7 @@
             "ref" : [ "string", "int", "float", "double", "boolean" ]
           }
         },
-        "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams",
+        "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r",
         "optional" : true
       }, {
         "name" : "hyperParams",
@@ -3603,7 +3641,7 @@
             }
           }
         },
-        "doc" : "Hyperparameters of the MLModel",
+        "doc" : "Hyperparameters of the MLModel\r",
         "optional" : true
       }, {
         "name" : "trainingMetrics",
@@ -3638,7 +3676,7 @@
             }
           }
         },
-        "doc" : "Metrics of the MLModel used in training",
+        "doc" : "Metrics of the MLModel used in training\r",
         "optional" : true
       }, {
         "name" : "onlineMetrics",
@@ -3646,7 +3684,7 @@
           "type" : "array",
           "items" : "MLMetric"
         },
-        "doc" : "Metrics of the MLModel used in production",
+        "doc" : "Metrics of the MLModel used in production\r",
         "optional" : true
       }, {
         "name" : "mlFeatures",
@@ -3654,7 +3692,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.MLFeatureUrn"
         },
-        "doc" : "List of features used for MLModel training",
+        "doc" : "List of features used for MLModel training\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -3669,7 +3707,7 @@
           "type" : "array",
           "items" : "string"
         },
-        "doc" : "Tags for the MLModel",
+        "doc" : "Tags for the MLModel\r",
         "default" : [ ]
       }, {
         "name" : "deployments",
@@ -3677,7 +3715,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "Deployments for the MLModel",
+        "doc" : "Deployments for the MLModel\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -3691,11 +3729,11 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "List of jobs (if any) used to train the model",
+        "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
-            "entityTypes" : [ "dataJob" ],
+            "entityTypes" : [ "dataJob", "dataProcessInstance" ],
             "isLineage" : true,
             "name" : "TrainedBy"
           }
@@ -3706,7 +3744,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "List of jobs (if any) that use the model",
+        "doc" : "List of jobs (if any) that use the model\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
@@ -3722,7 +3760,7 @@
           "type" : "array",
           "items" : "com.linkedin.common.Urn"
         },
-        "doc" : "Groups the model belongs to",
+        "doc" : "Groups the model belongs to\r",
         "optional" : true,
         "Relationship" : {
           "/*" : {
diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json
index c3e04add825c9..e375ac698ab51 100644
--- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json
+++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json
@@ -932,15 +932,18 @@
             "Searchable" : {
               "/actor" : {
                 "fieldName" : "tagAttributionActors",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/source" : {
                 "fieldName" : "tagAttributionSources",
-                "fieldType" : "URN"
+                "fieldType" : "URN",
+                "queryByDefault" : false
               },
               "/time" : {
                 "fieldName" : "tagAttributionDates",
-                "fieldType" : "DATETIME"
+                "fieldType" : "DATETIME",
+                "queryByDefault" : false
               }
             }
           } ]
@@ -1059,15 +1062,18 @@
       "Searchable" : {
         "/actor" : {
           "fieldName" : "termAttributionActors",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/source" : {
           "fieldName" : "termAttributionSources",
-          "fieldType" : "URN"
+          "fieldType" : "URN",
+          "queryByDefault" : false
         },
         "/time" : {
           "fieldName" : "termAttributionDates",
-          "fieldType" : "DATETIME"
+          "fieldType" : "DATETIME",
+          "queryByDefault" : false
         }
       }
     } ]
@@ -2777,7 +2783,7 @@
                     "Searchable" : {
                       "fieldName" : "managerLdap",
                       "fieldType" : "URN",
-                      "queryByDefault" : true
+                      "queryByDefault" : false
                     }
                   }, {
                     "name" : "departmentId",
@@ -3359,7 +3365,7 @@
                           "type" : "com.linkedin.dataset.SchemaFieldPath",
                           "doc" : "Flattened name of the field. Field is computed from jsonPath field.",
                           "Searchable" : {
-                            "boostScore" : 5.0,
+                            "boostScore" : 1.0,
                             "fieldName" : "fieldPaths",
                             "fieldType" : "TEXT",
                             "queryByDefault" : "true"
@@ -3533,15 +3539,18 @@
                           "Searchable" : {
                             "/tags/*/attribution/actor" : {
                               "fieldName" : "fieldTagAttributionActors",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/tags/*/attribution/source" : {
                               "fieldName" : "fieldTagAttributionSources",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/tags/*/attribution/time" : {
                               "fieldName" : "fieldTagAttributionDates",
-                              "fieldType" : "DATETIME"
+                              "fieldType" : "DATETIME",
+                              "queryByDefault" : false
                             },
                             "/tags/*/tag" : {
                               "boostScore" : 0.5,
@@ -3563,15 +3572,18 @@
                           "Searchable" : {
                             "/terms/*/attribution/actor" : {
                               "fieldName" : "fieldTermAttributionActors",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/terms/*/attribution/source" : {
                               "fieldName" : "fieldTermAttributionSources",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/terms/*/attribution/time" : {
                               "fieldName" : "fieldTermAttributionDates",
-                              "fieldType" : "DATETIME"
+                              "fieldType" : "DATETIME",
+                              "queryByDefault" : false
                             },
                             "/terms/*/urn" : {
                               "boostScore" : 0.5,
@@ -3744,11 +3756,13 @@
                           "Searchable" : {
                             "/tags/*/attribution/actor" : {
                               "fieldName" : "editedFieldTagAttributionActors",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/tags/*/attribution/source" : {
                               "fieldName" : "editedFieldTagAttributionSources",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/tags/*/attribution/time" : {
                               "fieldName" : "editedFieldTagAttributionDates",
@@ -3774,11 +3788,13 @@
                           "Searchable" : {
                             "/terms/*/attribution/actor" : {
                               "fieldName" : "editedFieldTermAttributionActors",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/terms/*/attribution/source" : {
                               "fieldName" : "editedFieldTermAttributionSources",
-                              "fieldType" : "URN"
+                              "fieldType" : "URN",
+                              "queryByDefault" : false
                             },
                             "/terms/*/attribution/time" : {
                               "fieldName" : "editedFieldTermAttributionDates",
@@ -3962,12 +3978,23 @@
                   "type" : "record",
                   "name" : "MLModelProperties",
                   "namespace" : "com.linkedin.ml.metadata",
-                  "doc" : "Properties associated with a ML Model",
+                  "doc" : "Properties associated with a ML Model\r",
                   "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ],
                   "fields" : [ {
+                    "name" : "name",
+                    "type" : "string",
+                    "doc" : "Display name of the MLModel\r",
+                    "optional" : true,
+                    "Searchable" : {
+                      "boostScore" : 10.0,
+                      "enableAutocomplete" : true,
+                      "fieldType" : "WORD_GRAM",
+                      "queryByDefault" : true
+                    }
+                  }, {
                     "name" : "description",
                     "type" : "string",
-                    "doc" : "Documentation of the MLModel",
+                    "doc" : "Documentation of the MLModel\r",
                     "optional" : true,
                     "Searchable" : {
                       "fieldType" : "TEXT",
@@ -3976,17 +4003,28 @@
                   }, {
                     "name" : "date",
                     "type" : "com.linkedin.common.Time",
-                    "doc" : "Date when the MLModel was developed",
+                    "doc" : "Date when the MLModel was developed\r",
+                    "optional" : true,
+                    "deprecated" : true
+                  }, {
+                    "name" : "created",
+                    "type" : "com.linkedin.common.TimeStamp",
+                    "doc" : "Audit stamp containing who created this and when\r",
+                    "optional" : true
+                  }, {
+                    "name" : "lastModified",
+                    "type" : "com.linkedin.common.TimeStamp",
+                    "doc" : "Date when the MLModel was last modified\r",
                     "optional" : true
                   }, {
                     "name" : "version",
                     "type" : "com.linkedin.common.VersionTag",
-                    "doc" : "Version of the MLModel",
+                    "doc" : "Version of the MLModel\r",
                     "optional" : true
                   }, {
                     "name" : "type",
                     "type" : "string",
-                    "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc",
+                    "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r",
                     "optional" : true,
                     "Searchable" : {
                       "fieldType" : "TEXT_PARTIAL"
@@ -4002,7 +4040,7 @@
                         "ref" : [ "string", "int", "float", "double", "boolean" ]
                       }
                     },
-                    "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams",
+                    "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r",
                     "optional" : true
                   }, {
                     "name" : "hyperParams",
@@ -4037,7 +4075,7 @@
                         }
                       }
                     },
-                    "doc" : "Hyperparameters of the MLModel",
+                    "doc" : "Hyperparameters of the MLModel\r",
                     "optional" : true
                   }, {
                     "name" : "trainingMetrics",
@@ -4072,7 +4110,7 @@
                         }
                       }
                     },
-                    "doc" : "Metrics of the MLModel used in training",
+                    "doc" : "Metrics of the MLModel used in training\r",
                     "optional" : true
                   }, {
                     "name" : "onlineMetrics",
@@ -4080,7 +4118,7 @@
                       "type" : "array",
                       "items" : "MLMetric"
                     },
-                    "doc" : "Metrics of the MLModel used in production",
+                    "doc" : "Metrics of the MLModel used in production\r",
                     "optional" : true
                   }, {
                     "name" : "mlFeatures",
@@ -4088,7 +4126,7 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.MLFeatureUrn"
                     },
-                    "doc" : "List of features used for MLModel training",
+                    "doc" : "List of features used for MLModel training\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
@@ -4103,7 +4141,7 @@
                       "type" : "array",
                       "items" : "string"
                     },
-                    "doc" : "Tags for the MLModel",
+                    "doc" : "Tags for the MLModel\r",
                     "default" : [ ]
                   }, {
                     "name" : "deployments",
@@ -4111,7 +4149,7 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.Urn"
                     },
-                    "doc" : "Deployments for the MLModel",
+                    "doc" : "Deployments for the MLModel\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
@@ -4125,11 +4163,11 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.Urn"
                     },
-                    "doc" : "List of jobs (if any) used to train the model",
+                    "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
-                        "entityTypes" : [ "dataJob" ],
+                        "entityTypes" : [ "dataJob", "dataProcessInstance" ],
                         "isLineage" : true,
                         "name" : "TrainedBy"
                       }
@@ -4140,7 +4178,7 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.Urn"
                     },
-                    "doc" : "List of jobs (if any) that use the model",
+                    "doc" : "List of jobs (if any) that use the model\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
@@ -4156,7 +4194,7 @@
                       "type" : "array",
                       "items" : "com.linkedin.common.Urn"
                     },
-                    "doc" : "Groups the model belongs to",
+                    "doc" : "Groups the model belongs to\r",
                     "optional" : true,
                     "Relationship" : {
                       "/*" : {
@@ -4959,12 +4997,23 @@
                   "type" : "record",
                   "name" : "MLModelGroupProperties",
                   "namespace" : "com.linkedin.ml.metadata",
-                  "doc" : "Properties associated with an ML Model Group",
+                  "doc" : "Properties associated with an ML Model Group\r",
                   "include" : [ "com.linkedin.common.CustomProperties" ],
                   "fields" : [ {
+                    "name" : "name",
+                    "type" : "string",
+                    "doc" : "Display name of the MLModelGroup\r",
+                    "optional" : true,
+                    "Searchable" : {
+                      "boostScore" : 10.0,
+                      "enableAutocomplete" : true,
+                      "fieldType" : "WORD_GRAM",
+                      "queryByDefault" : true
+                    }
+                  }, {
                     "name" : "description",
                     "type" : "string",
-                    "doc" : "Documentation of the MLModelGroup",
+                    "doc" : "Documentation of the MLModelGroup\r",
                     "optional" : true,
                     "Searchable" : {
                       "fieldType" : "TEXT",
@@ -4973,12 +5022,38 @@
                   }, {
                     "name" : "createdAt",
                     "type" : "com.linkedin.common.Time",
-                    "doc" : "Date when the MLModelGroup was developed",
+                    "doc" : "Date when the MLModelGroup was developed\r",
+                    "optional" : true,
+                    "deprecated" : true
+                  }, {
+                    "name" : "created",
+                    "type" : "com.linkedin.common.TimeStamp",
+                    "doc" : "Time and Actor who created the MLModelGroup\r",
+                    "optional" : true
+                  }, {
+                    "name" : "lastModified",
+                    "type" : "com.linkedin.common.TimeStamp",
+                    "doc" : "Date when the MLModelGroup was last modified\r",
                     "optional" : true
+                  }, {
+                    "name" : "trainingJobs",
+                    "type" : {
+                      "type" : "array",
+                      "items" : "com.linkedin.common.Urn"
+                    },
+                    "doc" : "List of jobs (if any) used to train the model group. Visible in Lineage.\r",
+                    "optional" : true,
+                    "Relationship" : {
+                      "/*" : {
+                        "entityTypes" : [ "dataJob" ],
+                        "isLineage" : true,
+                        "name" : "TrainedBy"
+                      }
+                    }
                   }, {
                     "name" : "version",
                     "type" : "com.linkedin.common.VersionTag",
-                    "doc" : "Version of the MLModelGroup",
+                    "doc" : "Version of the MLModelGroup\r",
                     "optional" : true
                   } ],
                   "Aspect" : {
@@ -5198,9 +5273,9 @@
                     },
                     "Searchable" : {
                       "/*" : {
-                        "boostScore" : 2.0,
                         "fieldName" : "isRelatedTerms",
-                        "fieldType" : "URN"
+                        "fieldType" : "URN",
+                        "queryByDefault" : false
                       }
                     }
                   }, {
@@ -5219,9 +5294,9 @@
                     },
                     "Searchable" : {
                       "/*" : {
-                        "boostScore" : 2.0,
                         "fieldName" : "hasRelatedTerms",
-                        "fieldType" : "URN"
+                        "fieldType" : "URN",
+                        "queryByDefault" : false
                       }
                     }
                   }, {
diff --git a/metadata-service/restli-client-api/build.gradle b/metadata-service/restli-client-api/build.gradle
index c2dfd4d2a1344..4e488a98023cb 100644
--- a/metadata-service/restli-client-api/build.gradle
+++ b/metadata-service/restli-client-api/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java-library'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   api project(path: ':metadata-service:restli-api', configuration: 'restClient')
   api project(':metadata-operation-context')
diff --git a/metadata-service/restli-client-api/src/main/java/com/linkedin/common/client/BaseClient.java b/metadata-service/restli-client-api/src/main/java/com/linkedin/common/client/BaseClient.java
index 4474fd5ce67ec..6cc4cf294d270 100644
--- a/metadata-service/restli-client-api/src/main/java/com/linkedin/common/client/BaseClient.java
+++ b/metadata-service/restli-client-api/src/main/java/com/linkedin/common/client/BaseClient.java
@@ -2,8 +2,8 @@
 
 import com.datahub.authentication.Authentication;
 import com.linkedin.common.callback.FutureCallback;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.metadata.utils.metrics.MetricUtils;
-import com.linkedin.parseq.retry.backoff.BackoffPolicy;
 import com.linkedin.r2.RemoteInvocationException;
 import com.linkedin.restli.client.AbstractRequestBuilder;
 import com.linkedin.restli.client.Client;
@@ -19,17 +19,15 @@
 @Slf4j
 public abstract class BaseClient implements AutoCloseable {
 
-  protected final Client _client;
-  protected final BackoffPolicy _backoffPolicy;
-  protected final int _retryCount;
+  protected final Client client;
+  protected final EntityClientConfig entityClientConfig;
 
   protected static final Set<String> NON_RETRYABLE =
       Set.of("com.linkedin.data.template.RequiredFieldNotPresentException");
 
-  protected BaseClient(@Nonnull Client restliClient, BackoffPolicy backoffPolicy, int retryCount) {
-    _client = Objects.requireNonNull(restliClient);
-    _backoffPolicy = backoffPolicy;
-    _retryCount = retryCount;
+  protected BaseClient(@Nonnull Client restliClient, EntityClientConfig entityClientConfig) {
+    client = Objects.requireNonNull(restliClient);
+    this.entityClientConfig = entityClientConfig;
   }
 
   protected <T> Response<T> sendClientRequest(
@@ -52,9 +50,9 @@ protected <T> Response<T> sendClientRequest(
 
     int attemptCount = 0;
 
-    while (attemptCount < _retryCount + 1) {
+    while (attemptCount < entityClientConfig.getRetryCount() + 1) {
       try {
-        return _client.sendRequest(requestBuilder.build()).getResponse();
+        return client.sendRequest(requestBuilder.build()).getResponse();
       } catch (Throwable ex) {
         MetricUtils.counter(
                 BaseClient.class,
@@ -66,12 +64,13 @@ protected <T> Response<T> sendClientRequest(
                 || (ex.getCause() != null
                     && NON_RETRYABLE.contains(ex.getCause().getClass().getCanonicalName()));
 
-        if (attemptCount == _retryCount || skipRetry) {
+        if (attemptCount == entityClientConfig.getRetryCount() || skipRetry) {
           throw ex;
         } else {
           attemptCount = attemptCount + 1;
           try {
-            Thread.sleep(_backoffPolicy.nextBackoff(attemptCount, ex) * 1000);
+            Thread.sleep(
+                entityClientConfig.getBackoffPolicy().nextBackoff(attemptCount, ex) * 1000);
           } catch (InterruptedException e) {
             throw new RuntimeException(e);
           }
@@ -84,6 +83,6 @@ protected <T> Response<T> sendClientRequest(
 
   @Override
   public void close() {
-    _client.shutdown(new FutureCallback<>());
+    client.shutdown(new FutureCallback<>());
   }
 }
diff --git a/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java
index 5f086e79a387a..b85f22e781d0b 100644
--- a/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java
+++ b/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java
@@ -36,6 +36,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nonnull;
@@ -44,12 +45,34 @@
 // Consider renaming this to datahub client.
 public interface EntityClient {
 
+  /**
+   * This version follows the legacy behavior of returning key aspects regardless of whether they
+   * exist
+   *
+   * @param opContext operation context
+   * @param entityName entity type
+   * @param urn urn id for the entity
+   * @param aspectNames set of aspects
+   * @return requested entity/aspects
+   */
+  @Deprecated
   @Nullable
-  EntityResponse getV2(
+  default EntityResponse getV2(
       @Nonnull OperationContext opContext,
       @Nonnull String entityName,
       @Nonnull final Urn urn,
       @Nullable final Set<String> aspectNames)
+      throws RemoteInvocationException, URISyntaxException {
+    return getV2(opContext, entityName, urn, aspectNames, true);
+  }
+
+  @Nullable
+  EntityResponse getV2(
+      @Nonnull OperationContext opContext,
+      @Nonnull String entityName,
+      @Nonnull final Urn urn,
+      @Nullable final Set<String> aspectNames,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException;
 
   @Nonnull
@@ -57,12 +80,34 @@ EntityResponse getV2(
   Entity get(@Nonnull OperationContext opContext, @Nonnull final Urn urn)
       throws RemoteInvocationException;
 
+  /**
+   * This version follows the legacy behavior of returning key aspects regardless of whether they
+   * exist
+   *
+   * @param opContext operation context
+   * @param entityName entity type
+   * @param urns urn ids for the entities
+   * @param aspectNames set of aspects
+   * @return requested entity/aspects
+   */
+  @Deprecated
+  @Nonnull
+  default Map<Urn, EntityResponse> batchGetV2(
+      @Nonnull OperationContext opContext,
+      @Nonnull String entityName,
+      @Nonnull final Set<Urn> urns,
+      @Nullable final Set<String> aspectNames)
+      throws RemoteInvocationException, URISyntaxException {
+    return batchGetV2(opContext, entityName, urns, aspectNames, true);
+  }
+
   @Nonnull
   Map<Urn, EntityResponse> batchGetV2(
       @Nonnull OperationContext opContext,
       @Nonnull String entityName,
       @Nonnull final Set<Urn> urns,
-      @Nullable final Set<String> aspectNames)
+      @Nullable final Set<String> aspectNames,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException;
 
   @Nonnull
@@ -523,12 +568,16 @@ default String ingestProposal(
    *
    * @return the urn string ingested
    */
+  @Nullable
   default String ingestProposal(
       @Nonnull OperationContext opContext,
       @Nonnull final MetadataChangeProposal metadataChangeProposal,
       final boolean async)
       throws RemoteInvocationException {
-    return batchIngestProposals(opContext, List.of(metadataChangeProposal), async).get(0);
+    return batchIngestProposals(opContext, List.of(metadataChangeProposal), async).stream()
+        .filter(Objects::nonNull)
+        .findFirst()
+        .orElse(null);
   }
 
   @Deprecated
@@ -584,27 +633,38 @@ void rollbackIngestion(
 
   @Nullable
   default Aspect getLatestAspectObject(
-      @Nonnull OperationContext opContext, @Nonnull Urn urn, @Nonnull String aspectName)
+      @Nonnull OperationContext opContext,
+      @Nonnull Urn urn,
+      @Nonnull String aspectName,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException {
-    return getLatestAspects(opContext, Set.of(urn), Set.of(aspectName))
+    return getLatestAspects(opContext, Set.of(urn), Set.of(aspectName), alwaysIncludeKeyAspect)
         .getOrDefault(urn, Map.of())
         .get(aspectName);
   }
 
   @Nonnull
   default Map<Urn, Map<String, Aspect>> getLatestAspects(
-      @Nonnull OperationContext opContext, @Nonnull Set<Urn> urns, @Nonnull Set<String> aspectNames)
+      @Nonnull OperationContext opContext,
+      @Nonnull Set<Urn> urns,
+      @Nonnull Set<String> aspectNames,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException {
     String entityName = urns.stream().findFirst().map(Urn::getEntityType).get();
-    return entityResponseToAspectMap(batchGetV2(opContext, entityName, urns, aspectNames));
+    return entityResponseToAspectMap(
+        batchGetV2(opContext, entityName, urns, aspectNames, alwaysIncludeKeyAspect));
   }
 
   @Nonnull
   default Map<Urn, Map<String, SystemAspect>> getLatestSystemAspect(
-      @Nonnull OperationContext opContext, @Nonnull Set<Urn> urns, @Nonnull Set<String> aspectNames)
+      @Nonnull OperationContext opContext,
+      @Nonnull Set<Urn> urns,
+      @Nonnull Set<String> aspectNames,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException {
     String entityName = urns.stream().findFirst().map(Urn::getEntityType).get();
     return entityResponseToSystemAspectMap(
-        batchGetV2(opContext, entityName, urns, aspectNames), opContext.getEntityRegistry());
+        batchGetV2(opContext, entityName, urns, aspectNames, alwaysIncludeKeyAspect),
+        opContext.getEntityRegistry());
   }
 }
diff --git a/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClientConfig.java b/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClientConfig.java
new file mode 100644
index 0000000000000..dab368f506d48
--- /dev/null
+++ b/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClientConfig.java
@@ -0,0 +1,46 @@
+package com.linkedin.entity.client;
+
+import com.linkedin.parseq.retry.backoff.BackoffPolicy;
+import com.linkedin.parseq.retry.backoff.ExponentialBackoff;
+import javax.annotation.Nonnull;
+import lombok.Builder;
+import lombok.Value;
+
+@Builder(toBuilder = true)
+@Value
+public class EntityClientConfig {
+  @Nonnull @Builder.Default BackoffPolicy backoffPolicy = new ExponentialBackoff(2);
+  @Builder.Default int retryCount = 3;
+  @Builder.Default int batchGetV2Size = 5;
+  @Builder.Default int batchGetV2Concurrency = 1;
+  @Builder.Default int batchGetV2QueueSize = 100;
+  @Builder.Default int batchGetV2KeepAlive = 60;
+  @Builder.Default int batchIngestSize = 5;
+  @Builder.Default int batchIngestConcurrency = 1;
+  @Builder.Default int batchIngestQueueSize = 100;
+  @Builder.Default int batchIngestKeepAlive = 60;
+
+  public int getBatchGetV2Size() {
+    return Math.max(1, batchGetV2Size);
+  }
+
+  public int getBatchGetV2Concurrency() {
+    return Math.max(1, batchGetV2Concurrency);
+  }
+
+  public int getBatchIngestSize() {
+    return Math.max(1, batchIngestSize);
+  }
+
+  public int getBatchIngestConcurrency() {
+    return Math.max(1, batchIngestConcurrency);
+  }
+
+  public int getBatchIngestQueueSize() {
+    return Math.max(1, batchIngestQueueSize);
+  }
+
+  public int getBatchGetV2QueueSize() {
+    return Math.max(1, batchGetV2QueueSize);
+  }
+}
diff --git a/metadata-service/restli-client/build.gradle b/metadata-service/restli-client/build.gradle
index 587ff5b2ef053..92b3433016633 100644
--- a/metadata-service/restli-client/build.gradle
+++ b/metadata-service/restli-client/build.gradle
@@ -3,6 +3,8 @@ plugins {
   id 'java-library'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   api project(':metadata-service:restli-api')
   api project(':metadata-auth:auth-api')
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java
index 824460b8a1a50..8d4c5e9228a71 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java
@@ -1,5 +1,6 @@
 package com.linkedin.entity.client;
 
+import static com.linkedin.metadata.Constants.RESTLI_SUCCESS;
 import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion;
 
 import com.datahub.plugins.auth.authorization.Authorizer;
@@ -74,7 +75,6 @@
 import com.linkedin.mxe.MetadataChangeProposalArray;
 import com.linkedin.mxe.PlatformEvent;
 import com.linkedin.mxe.SystemMetadata;
-import com.linkedin.parseq.retry.backoff.BackoffPolicy;
 import com.linkedin.platform.PlatformDoProducePlatformEventRequestBuilder;
 import com.linkedin.platform.PlatformRequestBuilders;
 import com.linkedin.r2.RemoteInvocationException;
@@ -83,17 +83,21 @@
 import com.linkedin.restli.common.HttpStatus;
 import io.datahubproject.metadata.context.OperationContext;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -120,18 +124,30 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
       new PlatformRequestBuilders();
   private static final RunsRequestBuilders RUNS_REQUEST_BUILDERS = new RunsRequestBuilders();
 
-  private final int batchGetV2Size;
-  private final int batchGetV2Concurrency;
+  private final ExecutorService batchGetV2Pool;
+  private final ExecutorService batchIngestPool;
 
   public RestliEntityClient(
-      @Nonnull final Client restliClient,
-      @Nonnull final BackoffPolicy backoffPolicy,
-      int retryCount,
-      int batchGetV2Size,
-      int batchGetV2Concurrency) {
-    super(restliClient, backoffPolicy, retryCount);
-    this.batchGetV2Size = Math.max(1, batchGetV2Size);
-    this.batchGetV2Concurrency = batchGetV2Concurrency;
+      @Nonnull final Client restliClient, EntityClientConfig entityClientConfig) {
+    super(restliClient, entityClientConfig);
+    this.batchGetV2Pool =
+        new ThreadPoolExecutor(
+            entityClientConfig.getBatchGetV2Concurrency(), // core threads
+            entityClientConfig.getBatchGetV2Concurrency(), // max threads
+            entityClientConfig.getBatchGetV2KeepAlive(),
+            TimeUnit.SECONDS, // thread keep-alive time
+            new ArrayBlockingQueue<>(
+                entityClientConfig.getBatchGetV2QueueSize()), // fixed size queue
+            new ThreadPoolExecutor.CallerRunsPolicy());
+    this.batchIngestPool =
+        new ThreadPoolExecutor(
+            entityClientConfig.getBatchIngestConcurrency(), // core threads
+            entityClientConfig.getBatchIngestConcurrency(), // max threads
+            entityClientConfig.getBatchIngestKeepAlive(),
+            TimeUnit.SECONDS, // thread keep-alive time
+            new ArrayBlockingQueue<>(
+                entityClientConfig.getBatchIngestQueueSize()), // fixed size queue
+            new ThreadPoolExecutor.CallerRunsPolicy());
   }
 
   @Override
@@ -140,10 +156,15 @@ public EntityResponse getV2(
       @Nonnull OperationContext opContext,
       @Nonnull String entityName,
       @Nonnull final Urn urn,
-      @Nullable final Set<String> aspectNames)
+      @Nullable final Set<String> aspectNames,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException {
     final EntitiesV2GetRequestBuilder requestBuilder =
-        ENTITIES_V2_REQUEST_BUILDERS.get().aspectsParam(aspectNames).id(urn.toString());
+        ENTITIES_V2_REQUEST_BUILDERS
+            .get()
+            .aspectsParam(aspectNames)
+            .id(urn.toString())
+            .alwaysIncludeKeyAspectParam(alwaysIncludeKeyAspect);
     return sendClientRequest(requestBuilder, opContext.getSessionAuthentication()).getEntity();
   }
 
@@ -225,58 +246,56 @@ public Map<Urn, EntityResponse> batchGetV2(
       @Nonnull OperationContext opContext,
       @Nonnull String entityName,
       @Nonnull final Set<Urn> urns,
-      @Nullable final Set<String> aspectNames)
+      @Nullable final Set<String> aspectNames,
+      @Nullable Boolean alwaysIncludeKeyAspect)
       throws RemoteInvocationException, URISyntaxException {
 
     Map<Urn, EntityResponse> responseMap = new HashMap<>();
-    ExecutorService executor = Executors.newFixedThreadPool(Math.max(1, batchGetV2Concurrency));
 
-    try {
-      Iterable<List<Urn>> iterable = () -> Iterators.partition(urns.iterator(), batchGetV2Size);
-      List<Future<Map<Urn, EntityResponse>>> futures =
-          StreamSupport.stream(iterable.spliterator(), false)
-              .map(
-                  batch ->
-                      executor.submit(
-                          () -> {
-                            try {
-                              log.debug("Executing batchGetV2 with batch size: {}", batch.size());
-                              final EntitiesV2BatchGetRequestBuilder requestBuilder =
-                                  ENTITIES_V2_REQUEST_BUILDERS
-                                      .batchGet()
-                                      .aspectsParam(aspectNames)
-                                      .ids(
-                                          batch.stream()
-                                              .map(Urn::toString)
-                                              .collect(Collectors.toList()));
-
-                              return sendClientRequest(
-                                      requestBuilder, opContext.getSessionAuthentication())
-                                  .getEntity()
-                                  .getResults()
-                                  .entrySet()
-                                  .stream()
-                                  .collect(
-                                      Collectors.toMap(
-                                          entry -> UrnUtils.getUrn(entry.getKey()),
-                                          entry -> entry.getValue().getEntity()));
-                            } catch (RemoteInvocationException e) {
-                              throw new RuntimeException(e);
-                            }
-                          }))
-              .collect(Collectors.toList());
-
-      futures.forEach(
-          result -> {
-            try {
-              responseMap.putAll(result.get());
-            } catch (InterruptedException | ExecutionException e) {
-              throw new RuntimeException(e);
-            }
-          });
-    } finally {
-      executor.shutdown();
-    }
+    Iterable<List<Urn>> iterable =
+        () -> Iterators.partition(urns.iterator(), entityClientConfig.getBatchGetV2Size());
+    List<Future<Map<Urn, EntityResponse>>> futures =
+        StreamSupport.stream(iterable.spliterator(), false)
+            .map(
+                batch ->
+                    batchGetV2Pool.submit(
+                        () -> {
+                          try {
+                            log.debug("Executing batchGetV2 with batch size: {}", batch.size());
+                            final EntitiesV2BatchGetRequestBuilder requestBuilder =
+                                ENTITIES_V2_REQUEST_BUILDERS
+                                    .batchGet()
+                                    .aspectsParam(aspectNames)
+                                    .alwaysIncludeKeyAspectParam(alwaysIncludeKeyAspect)
+                                    .ids(
+                                        batch.stream()
+                                            .map(Urn::toString)
+                                            .collect(Collectors.toList()));
+
+                            return sendClientRequest(
+                                    requestBuilder, opContext.getSessionAuthentication())
+                                .getEntity()
+                                .getResults()
+                                .entrySet()
+                                .stream()
+                                .collect(
+                                    Collectors.toMap(
+                                        entry -> UrnUtils.getUrn(entry.getKey()),
+                                        entry -> entry.getValue().getEntity()));
+                          } catch (RemoteInvocationException e) {
+                            throw new RuntimeException(e);
+                          }
+                        }))
+            .collect(Collectors.toList());
+
+    futures.forEach(
+        result -> {
+          try {
+            responseMap.putAll(result.get());
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException(e);
+          }
+        });
 
     return responseMap;
   }
@@ -298,62 +317,56 @@ public Map<Urn, EntityResponse> batchGetVersionedV2(
       @Nullable final Set<String> aspectNames) {
 
     Map<Urn, EntityResponse> responseMap = new HashMap<>();
-    ExecutorService executor = Executors.newFixedThreadPool(Math.max(1, batchGetV2Concurrency));
 
-    try {
-      Iterable<List<VersionedUrn>> iterable =
-          () -> Iterators.partition(versionedUrns.iterator(), batchGetV2Size);
-      List<Future<Map<Urn, EntityResponse>>> futures =
-          StreamSupport.stream(iterable.spliterator(), false)
-              .map(
-                  batch ->
-                      executor.submit(
-                          () -> {
-                            try {
-                              log.debug(
-                                  "Executing batchGetVersionedV2 with batch size: {}",
-                                  batch.size());
-                              final EntitiesVersionedV2BatchGetRequestBuilder requestBuilder =
-                                  ENTITIES_VERSIONED_V2_REQUEST_BUILDERS
-                                      .batchGet()
-                                      .aspectsParam(aspectNames)
-                                      .entityTypeParam(entityName)
-                                      .ids(
-                                          batch.stream()
-                                              .map(
-                                                  versionedUrn ->
-                                                      com.linkedin.common.urn.VersionedUrn.of(
-                                                          versionedUrn.getUrn().toString(),
-                                                          versionedUrn.getVersionStamp()))
-                                              .collect(Collectors.toSet()));
-
-                              return sendClientRequest(
-                                      requestBuilder, opContext.getSessionAuthentication())
-                                  .getEntity()
-                                  .getResults()
-                                  .entrySet()
-                                  .stream()
-                                  .collect(
-                                      Collectors.toMap(
-                                          entry -> UrnUtils.getUrn(entry.getKey().getUrn()),
-                                          entry -> entry.getValue().getEntity()));
-                            } catch (RemoteInvocationException e) {
-                              throw new RuntimeException(e);
-                            }
-                          }))
-              .collect(Collectors.toList());
-
-      futures.forEach(
-          result -> {
-            try {
-              responseMap.putAll(result.get());
-            } catch (InterruptedException | ExecutionException e) {
-              throw new RuntimeException(e);
-            }
-          });
-    } finally {
-      executor.shutdown();
-    }
+    Iterable<List<VersionedUrn>> iterable =
+        () -> Iterators.partition(versionedUrns.iterator(), entityClientConfig.getBatchGetV2Size());
+    List<Future<Map<Urn, EntityResponse>>> futures =
+        StreamSupport.stream(iterable.spliterator(), false)
+            .map(
+                batch ->
+                    batchGetV2Pool.submit(
+                        () -> {
+                          try {
+                            log.debug(
+                                "Executing batchGetVersionedV2 with batch size: {}", batch.size());
+                            final EntitiesVersionedV2BatchGetRequestBuilder requestBuilder =
+                                ENTITIES_VERSIONED_V2_REQUEST_BUILDERS
+                                    .batchGet()
+                                    .aspectsParam(aspectNames)
+                                    .entityTypeParam(entityName)
+                                    .ids(
+                                        batch.stream()
+                                            .map(
+                                                versionedUrn ->
+                                                    com.linkedin.common.urn.VersionedUrn.of(
+                                                        versionedUrn.getUrn().toString(),
+                                                        versionedUrn.getVersionStamp()))
+                                            .collect(Collectors.toSet()));
+
+                            return sendClientRequest(
+                                    requestBuilder, opContext.getSessionAuthentication())
+                                .getEntity()
+                                .getResults()
+                                .entrySet()
+                                .stream()
+                                .collect(
+                                    Collectors.toMap(
+                                        entry -> UrnUtils.getUrn(entry.getKey().getUrn()),
+                                        entry -> entry.getValue().getEntity()));
+                          } catch (RemoteInvocationException e) {
+                            throw new RuntimeException(e);
+                          }
+                        }))
+            .collect(Collectors.toList());
+
+    futures.forEach(
+        result -> {
+          try {
+            responseMap.putAll(result.get());
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException(e);
+          }
+        });
 
     return responseMap;
   }
@@ -1075,29 +1088,68 @@ public List<String> batchIngestProposals(
       @Nonnull Collection<MetadataChangeProposal> metadataChangeProposals,
       boolean async)
       throws RemoteInvocationException {
-    final AspectsDoIngestProposalBatchRequestBuilder requestBuilder =
-        ASPECTS_REQUEST_BUILDERS
-            .actionIngestProposalBatch()
-            .proposalsParam(new MetadataChangeProposalArray(metadataChangeProposals))
-            .asyncParam(String.valueOf(async));
-    String result =
-        sendClientRequest(requestBuilder, opContext.getSessionAuthentication()).getEntity();
-    return metadataChangeProposals.stream()
-        .map(
-            proposal -> {
-              if ("success".equals(result)) {
-                if (proposal.getEntityUrn() != null) {
-                  return proposal.getEntityUrn().toString();
-                } else {
-                  EntitySpec entitySpec =
-                      opContext.getEntityRegistry().getEntitySpec(proposal.getEntityType());
-                  return EntityKeyUtils.getUrnFromProposal(proposal, entitySpec.getKeyAspectSpec())
-                      .toString();
-                }
-              }
-              return null;
-            })
-        .collect(Collectors.toList());
+
+    List<String> response = new ArrayList<>();
+
+    Iterable<List<MetadataChangeProposal>> iterable =
+        () ->
+            Iterators.partition(
+                metadataChangeProposals.iterator(), entityClientConfig.getBatchIngestSize());
+    List<Future<List<String>>> futures =
+        StreamSupport.stream(iterable.spliterator(), false)
+            .map(
+                batch ->
+                    batchIngestPool.submit(
+                        () -> {
+                          try {
+                            log.debug(
+                                "Executing batchIngestProposals with batch size: {}", batch.size());
+                            final AspectsDoIngestProposalBatchRequestBuilder requestBuilder =
+                                ASPECTS_REQUEST_BUILDERS
+                                    .actionIngestProposalBatch()
+                                    .proposalsParam(
+                                        new MetadataChangeProposalArray(metadataChangeProposals))
+                                    .asyncParam(String.valueOf(async));
+                            String result =
+                                sendClientRequest(
+                                        requestBuilder, opContext.getSessionAuthentication())
+                                    .getEntity();
+
+                            if (RESTLI_SUCCESS.equals(result)) {
+                              return batch.stream()
+                                  .map(
+                                      mcp -> {
+                                        if (mcp.getEntityUrn() != null) {
+                                          return mcp.getEntityUrn().toString();
+                                        } else {
+                                          EntitySpec entitySpec =
+                                              opContext
+                                                  .getEntityRegistry()
+                                                  .getEntitySpec(mcp.getEntityType());
+                                          return EntityKeyUtils.getUrnFromProposal(
+                                                  mcp, entitySpec.getKeyAspectSpec())
+                                              .toString();
+                                        }
+                                      })
+                                  .collect(Collectors.toList());
+                            }
+                            return Collections.<String>emptyList();
+                          } catch (RemoteInvocationException e) {
+                            throw new RuntimeException(e);
+                          }
+                        }))
+            .collect(Collectors.toList());
+
+    futures.forEach(
+        result -> {
+          try {
+            response.addAll(result.get());
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException(e);
+          }
+        });
+
+    return response;
   }
 
   @Override
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java
index 7546d1f0a3b54..aa17f1951bc91 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java
@@ -5,7 +5,6 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig;
-import com.linkedin.parseq.retry.backoff.BackoffPolicy;
 import com.linkedin.r2.RemoteInvocationException;
 import com.linkedin.restli.client.Client;
 import io.datahubproject.metadata.context.OperationContext;
@@ -24,12 +23,9 @@ public class SystemRestliEntityClient extends RestliEntityClient implements Syst
 
   public SystemRestliEntityClient(
       @Nonnull final Client restliClient,
-      @Nonnull final BackoffPolicy backoffPolicy,
-      int retryCount,
-      EntityClientCacheConfig cacheConfig,
-      int batchGetV2Size,
-      int batchGetV2Concurrency) {
-    super(restliClient, backoffPolicy, retryCount, batchGetV2Size, batchGetV2Concurrency);
+      @Nonnull EntityClientConfig clientConfig,
+      EntityClientCacheConfig cacheConfig) {
+    super(restliClient, clientConfig);
     this.operationContextMap = CacheBuilder.newBuilder().maximumSize(500).build();
     this.entityClientCache = buildEntityClientCache(SystemRestliEntityClient.class, cacheConfig);
   }
@@ -63,6 +59,6 @@ public Map<Urn, EntityResponse> batchGetV2NoCache(
       @Nonnull Set<Urn> urns,
       @Nullable Set<String> aspectNames)
       throws RemoteInvocationException, URISyntaxException {
-    return super.batchGetV2(opContext, entityName, urns, aspectNames);
+    return super.batchGetV2(opContext, entityName, urns, aspectNames, false);
   }
 }
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/usage/RestliUsageClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/usage/RestliUsageClient.java
index e55d9b29f1d2f..e50c7badb5b9f 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/usage/RestliUsageClient.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/usage/RestliUsageClient.java
@@ -5,6 +5,7 @@
 import com.linkedin.common.EntityRelationships;
 import com.linkedin.common.WindowDuration;
 import com.linkedin.common.client.BaseClient;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.metadata.config.cache.client.UsageClientCacheConfig;
 import com.linkedin.parseq.retry.backoff.BackoffPolicy;
 import com.linkedin.r2.RemoteInvocationException;
@@ -26,7 +27,9 @@ public RestliUsageClient(
       @Nonnull final BackoffPolicy backoffPolicy,
       int retryCount,
       UsageClientCacheConfig cacheConfig) {
-    super(restliClient, backoffPolicy, retryCount);
+    super(
+        restliClient,
+        EntityClientConfig.builder().backoffPolicy(backoffPolicy).retryCount(retryCount).build());
     this.operationContextMap = Caffeine.newBuilder().maximumSize(500).build();
     this.usageClientCache =
         UsageClientCache.builder()
diff --git a/metadata-service/restli-client/src/test/java/com/linkedin/common/client/BaseClientTest.java b/metadata-service/restli-client/src/test/java/com/linkedin/common/client/BaseClientTest.java
index 797ead10c1a66..285b57972ece4 100644
--- a/metadata-service/restli-client/src/test/java/com/linkedin/common/client/BaseClientTest.java
+++ b/metadata-service/restli-client/src/test/java/com/linkedin/common/client/BaseClientTest.java
@@ -13,6 +13,7 @@
 import com.linkedin.data.template.RequiredFieldNotPresentException;
 import com.linkedin.entity.AspectsDoIngestProposalRequestBuilder;
 import com.linkedin.entity.AspectsRequestBuilders;
+import com.linkedin.entity.client.EntityClientConfig;
 import com.linkedin.entity.client.RestliEntityClient;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.parseq.retry.backoff.ExponentialBackoff;
@@ -37,7 +38,14 @@ public void testZeroRetry() throws RemoteInvocationException {
     when(mockRestliClient.sendRequest(any(ActionRequest.class))).thenReturn(mockFuture);
 
     RestliEntityClient testClient =
-        new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 0, 10, 2);
+        new RestliEntityClient(
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ExponentialBackoff(1))
+                .retryCount(0)
+                .batchGetV2Size(10)
+                .batchGetV2Concurrency(2)
+                .build());
     testClient.sendClientRequest(testRequestBuilder, AUTH);
     // Expected 1 actual try and 0 retries
     verify(mockRestliClient).sendRequest(any(ActionRequest.class));
@@ -56,7 +64,14 @@ public void testMultipleRetries() throws RemoteInvocationException {
         .thenReturn(mockFuture);
 
     RestliEntityClient testClient =
-        new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 1, 10, 2);
+        new RestliEntityClient(
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ExponentialBackoff(1))
+                .retryCount(1)
+                .batchGetV2Size(10)
+                .batchGetV2Concurrency(2)
+                .build());
     testClient.sendClientRequest(testRequestBuilder, AUTH);
     // Expected 1 actual try and 1 retries
     verify(mockRestliClient, times(2)).sendRequest(any(ActionRequest.class));
@@ -73,7 +88,14 @@ public void testNonRetry() {
         .thenThrow(new RuntimeException(new RequiredFieldNotPresentException("value")));
 
     RestliEntityClient testClient =
-        new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 1, 10, 2);
+        new RestliEntityClient(
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ExponentialBackoff(1))
+                .retryCount(1)
+                .batchGetV2Size(10)
+                .batchGetV2Concurrency(2)
+                .build());
     assertThrows(
         RuntimeException.class, () -> testClient.sendClientRequest(testRequestBuilder, AUTH));
   }
diff --git a/metadata-service/restli-client/src/test/java/com/linkedin/entity/client/SystemRestliEntityClientTest.java b/metadata-service/restli-client/src/test/java/com/linkedin/entity/client/SystemRestliEntityClientTest.java
index 817b76f74268c..0449b3dfa523b 100644
--- a/metadata-service/restli-client/src/test/java/com/linkedin/entity/client/SystemRestliEntityClientTest.java
+++ b/metadata-service/restli-client/src/test/java/com/linkedin/entity/client/SystemRestliEntityClientTest.java
@@ -47,7 +47,14 @@ public void testCache() throws RemoteInvocationException, URISyntaxException {
 
     SystemRestliEntityClient noCacheTest =
         new SystemRestliEntityClient(
-            mockRestliClient, new ConstantBackoff(0), 0, noCacheConfig, 1, 2);
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ConstantBackoff(0))
+                .retryCount(0)
+                .batchGetV2Size(1)
+                .batchGetV2Concurrency(2)
+                .build(),
+            noCacheConfig);
 
     com.linkedin.entity.EntityResponse responseStatusTrue = buildStatusResponse(true);
     com.linkedin.entity.EntityResponse responseStatusFalse = buildStatusResponse(false);
@@ -86,7 +93,14 @@ public void testCache() throws RemoteInvocationException, URISyntaxException {
 
     SystemRestliEntityClient cacheTest =
         new SystemRestliEntityClient(
-            mockRestliClient, new ConstantBackoff(0), 0, cacheConfig, 1, 2);
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ConstantBackoff(0))
+                .retryCount(0)
+                .batchGetV2Size(1)
+                .batchGetV2Concurrency(2)
+                .build(),
+            cacheConfig);
 
     mockResponse(mockRestliClient, responseStatusTrue);
     assertEquals(
@@ -121,7 +135,14 @@ public void testBatchCache() throws RemoteInvocationException, URISyntaxExceptio
 
     SystemRestliEntityClient noCacheTest =
         new SystemRestliEntityClient(
-            mockRestliClient, new ConstantBackoff(0), 0, noCacheConfig, 1, 2);
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ConstantBackoff(0))
+                .retryCount(0)
+                .batchGetV2Size(1)
+                .batchGetV2Concurrency(2)
+                .build(),
+            noCacheConfig);
 
     com.linkedin.entity.EntityResponse responseStatusTrue = buildStatusResponse(true);
     com.linkedin.entity.EntityResponse responseStatusFalse = buildStatusResponse(false);
@@ -160,7 +181,14 @@ public void testBatchCache() throws RemoteInvocationException, URISyntaxExceptio
 
     SystemRestliEntityClient cacheTest =
         new SystemRestliEntityClient(
-            mockRestliClient, new ConstantBackoff(0), 0, cacheConfig, 1, 2);
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ConstantBackoff(0))
+                .retryCount(0)
+                .batchGetV2Size(1)
+                .batchGetV2Concurrency(2)
+                .build(),
+            cacheConfig);
 
     mockResponse(mockRestliClient, responseStatusTrue);
     assertEquals(
@@ -195,7 +223,14 @@ public void testCacheNullValue() throws RemoteInvocationException, URISyntaxExce
 
     SystemRestliEntityClient noCacheTest =
         new SystemRestliEntityClient(
-            mockRestliClient, new ConstantBackoff(0), 0, noCacheConfig, 1, 2);
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ConstantBackoff(0))
+                .retryCount(0)
+                .batchGetV2Size(1)
+                .batchGetV2Concurrency(2)
+                .build(),
+            noCacheConfig);
 
     com.linkedin.entity.EntityResponse responseStatusTrue = buildStatusResponse(true);
     com.linkedin.entity.EntityResponse responseStatusFalse = buildStatusResponse(false);
@@ -238,7 +273,14 @@ public void testCacheNullValue() throws RemoteInvocationException, URISyntaxExce
 
     SystemRestliEntityClient cacheTest =
         new SystemRestliEntityClient(
-            mockRestliClient, new ConstantBackoff(0), 0, cacheConfig, 1, 2);
+            mockRestliClient,
+            EntityClientConfig.builder()
+                .backoffPolicy(new ConstantBackoff(0))
+                .retryCount(0)
+                .batchGetV2Size(1)
+                .batchGetV2Concurrency(2)
+                .build(),
+            cacheConfig);
 
     mockResponse(mockRestliClient, responseStatusTrue);
     assertEquals(
diff --git a/metadata-service/restli-servlet-impl/build.gradle b/metadata-service/restli-servlet-impl/build.gradle
index 6b68abfe7fb15..dc426bf5639b1 100644
--- a/metadata-service/restli-servlet-impl/build.gradle
+++ b/metadata-service/restli-servlet-impl/build.gradle
@@ -3,6 +3,8 @@ plugins {
   id 'pegasus'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 sourceSets {
   integTest {
     compileClasspath += sourceSets.main.output
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java
index 9bbe1bb35fc65..94da6308eda1f 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java
@@ -5,7 +5,6 @@
 import com.datahub.authorization.AuthUtil;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.linkedin.analytics.GetTimeseriesAggregatedStatsResponse;
-import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.resources.restli.RestliUtils;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
@@ -14,7 +13,6 @@
 import com.linkedin.restli.server.RestLiServiceException;
 import com.linkedin.restli.server.annotations.Action;
 import com.linkedin.restli.server.annotations.ActionParam;
-import com.linkedin.restli.server.annotations.Context;
 import com.linkedin.restli.server.annotations.Optional;
 import com.linkedin.restli.server.annotations.RestLiSimpleResource;
 import com.linkedin.restli.server.resources.SimpleResourceTemplate;
@@ -24,12 +22,10 @@
 import com.linkedin.timeseries.GroupingBucket;
 import com.linkedin.timeseries.GroupingBucketArray;
 import java.util.Arrays;
-import java.util.List;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.inject.Named;
-import javax.servlet.http.HttpServletRequest;
 
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.RequestContext;
@@ -38,6 +34,7 @@
 import static com.datahub.authorization.AuthUtil.isAPIAuthorized;
 import static com.linkedin.metadata.authorization.ApiGroup.TIMESERIES;
 import static com.linkedin.metadata.authorization.ApiOperation.READ;
+import static com.linkedin.metadata.utils.CriterionUtils.validateAndConvert;
 
 /** Rest.li entry point: /analytics */
 @Slf4j
@@ -90,8 +87,9 @@ public Task<GetTimeseriesAggregatedStatsResponse> getTimeseriesStats(
           resp.setEntityName(entityName);
           resp.setAspectName(aspectName);
           resp.setAggregationSpecs(new AggregationSpecArray(Arrays.asList(aggregationSpecs)));
-          if (filter != null) {
-            resp.setFilter(filter);
+          final Filter finalFilter = validateAndConvert(filter);
+          if (finalFilter != null) {
+            resp.setFilter(finalFilter);
           }
           if (groupingBuckets != null) {
             resp.setGroupingBuckets(new GroupingBucketArray(Arrays.asList(groupingBuckets)));
@@ -99,7 +97,7 @@ public Task<GetTimeseriesAggregatedStatsResponse> getTimeseriesStats(
 
           GenericTable aggregatedStatsTable =
               timeseriesAspectService.getAggregatedStats(opContext,
-                  entityName, aspectName, aggregationSpecs, filter, groupingBuckets);
+                  entityName, aspectName, aggregationSpecs, finalFilter, groupingBuckets);
           resp.setTable(aggregatedStatsTable);
           return resp;
         });
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
index 12422044bc8ef..30b187da00e91 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
@@ -4,12 +4,14 @@
 import static com.datahub.authorization.AuthUtil.isAPIAuthorizedEntityUrns;
 import static com.datahub.authorization.AuthUtil.isAPIAuthorizedUrns;
 import static com.datahub.authorization.AuthUtil.isAPIOperationsAuthorized;
+import static com.linkedin.metadata.Constants.RESTLI_SUCCESS;
 import static com.linkedin.metadata.authorization.ApiGroup.COUNTS;
 import static com.linkedin.metadata.authorization.ApiGroup.ENTITY;
 import static com.linkedin.metadata.authorization.ApiGroup.TIMESERIES;
 import static com.linkedin.metadata.authorization.ApiOperation.READ;
 import static com.linkedin.metadata.resources.operations.OperationsResource.*;
 import static com.linkedin.metadata.resources.restli.RestliConstants.*;
+import static com.linkedin.metadata.utils.CriterionUtils.validateAndConvert;
 
 import com.codahale.metrics.MetricRegistry;
 import com.datahub.authentication.Authentication;
@@ -33,8 +35,8 @@
 import com.linkedin.metadata.resources.restli.RestliUtils;
 import com.linkedin.metadata.search.EntitySearchService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
+import com.linkedin.mxe.GenericAspect;
 import com.linkedin.mxe.MetadataChangeProposal;
-import com.linkedin.mxe.SystemMetadata;
 import com.linkedin.parseq.Task;
 import com.linkedin.restli.common.HttpStatus;
 import com.linkedin.restli.internal.server.methods.AnyRecord;
@@ -51,6 +53,7 @@
 import io.datahubproject.metadata.context.RequestContext;
 import io.opentelemetry.extension.annotations.WithSpan;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.time.Clock;
 import java.util.Arrays;
 import java.util.List;
@@ -61,6 +64,7 @@
 import javax.inject.Inject;
 import javax.inject.Named;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 
 /** Single unified resource for fetching, updating, searching, & browsing DataHub entities */
 @Slf4j
@@ -85,6 +89,8 @@ public class AspectResource extends CollectionResourceTaskTemplate<String, Versi
 
   private final Clock _clock = Clock.systemUTC();
 
+  private static final int MAX_LOG_WIDTH = 512;
+
   @Inject
   @Named("entityService")
   private EntityService<?> _entityService;
@@ -120,6 +126,11 @@ void setSystemOperationContext(OperationContext systemOperationContext) {
       this.systemOperationContext = systemOperationContext;
   }
 
+  @VisibleForTesting
+  void setEntitySearchService(EntitySearchService entitySearchService) {
+    this.entitySearchService = entitySearchService;
+  }
+
   /**
    * Retrieves the value for an entity that is made up of latest versions of specified aspects.
    * TODO: Get rid of this and migrate to getAspect.
@@ -224,7 +235,7 @@ public Task<GetTimeseriesAspectValuesResponse> getTimeseriesAspectValues(
                       startTimeMillis,
                       endTimeMillis,
                       limit,
-                      filter,
+                      validateAndConvert(filter),
                       sort)));
           return response;
         },
@@ -238,16 +249,13 @@ public Task<String> ingestProposal(
       @ActionParam(PARAM_PROPOSAL) @Nonnull MetadataChangeProposal metadataChangeProposal,
       @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async)
       throws URISyntaxException {
-      log.info("INGEST PROPOSAL proposal: {}", metadataChangeProposal);
-
-      final boolean asyncBool;
-      if (UNSET.equals(async)) {
-          asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));
-      } else {
-          asyncBool = Boolean.parseBoolean(async);
-      }
-
-      return ingestProposals(List.of(metadataChangeProposal), asyncBool);
+    final boolean asyncBool;
+    if (UNSET.equals(async)) {
+      asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));
+    } else {
+      asyncBool = Boolean.parseBoolean(async);
+    }
+    return ingestProposals(List.of(metadataChangeProposal), asyncBool);
   }
 
   @Action(name = ACTION_INGEST_PROPOSAL_BATCH)
@@ -257,8 +265,6 @@ public Task<String> ingestProposalBatch(
             @ActionParam(PARAM_PROPOSALS) @Nonnull MetadataChangeProposal[] metadataChangeProposals,
             @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async)
             throws URISyntaxException {
-        log.info("INGEST PROPOSAL BATCH proposals: {}", Arrays.asList(metadataChangeProposals));
-
         final boolean asyncBool;
         if (UNSET.equals(async)) {
             asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));
@@ -275,12 +281,13 @@ private Task<String> ingestProposals(
           boolean asyncBool)
   throws URISyntaxException {
     Authentication authentication = AuthenticationContext.getAuthentication();
+    String actorUrnStr = authentication.getActor().toUrnStr();
 
     Set<String> entityTypes = metadataChangeProposals.stream()
                                                      .map(MetadataChangeProposal::getEntityType)
                                                      .collect(Collectors.toSet());
     final OperationContext opContext = OperationContext.asSession(
-              systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), getContext(),
+              systemOperationContext, RequestContext.builder().buildRestli(actorUrnStr, getContext(),
                     ACTION_INGEST_PROPOSAL, entityTypes), _authorizer, authentication, true);
 
     // Ingest Authorization Checks
@@ -293,9 +300,8 @@ private Task<String> ingestProposals(
                  .map(ex -> String.format("HttpStatus: %s Urn: %s", ex.getSecond(), ex.getFirst().getEntityUrn()))
                  .collect(Collectors.joining(", "));
         throw new RestLiServiceException(
-                 HttpStatus.S_403_FORBIDDEN, "User is unauthorized to modify entity: " + errorMessages);
+                 HttpStatus.S_403_FORBIDDEN, "User " + actorUrnStr + " is unauthorized to modify entity: " + errorMessages);
     }
-    String actorUrnStr = authentication.getActor().toUrnStr();
     final AuditStamp auditStamp =
         new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr));
 
@@ -303,23 +309,26 @@ private Task<String> ingestProposals(
       log.debug("Proposals: {}", metadataChangeProposals);
       try {
         final AspectsBatch batch = AspectsBatchImpl.builder()
-                .mcps(metadataChangeProposals, auditStamp, opContext.getRetrieverContext().get())
+                .mcps(metadataChangeProposals, auditStamp, opContext.getRetrieverContext(),
+                    opContext.getValidationContext().isAlternateValidation())
                 .build();
 
-        Set<IngestResult> results =
+        batch.getMCPItems().forEach(item ->
+            log.info(
+                    "INGEST PROPOSAL content: urn: {}, async: {}, value: {}",
+                    item.getUrn(),
+                    asyncBool,
+                    StringUtils.abbreviate(java.util.Optional.ofNullable(item.getMetadataChangeProposal())
+                            .map(MetadataChangeProposal::getAspect)
+                            .orElse(new GenericAspect())
+                            .getValue().asString(StandardCharsets.UTF_8), MAX_LOG_WIDTH)));
+
+        List<IngestResult> results =
                 _entityService.ingestProposal(opContext, batch, asyncBool);
-
-            for (IngestResult result : results) {
-                // Update runIds, only works for existing documents, so ES document must exist
-                Urn resultUrn = result.getUrn();
-
-                if (resultUrn != null && (result.isProcessedMCL() || result.isUpdate())) {
-                    tryIndexRunId(opContext, resultUrn, result.getRequest().getSystemMetadata(), entitySearchService);
-                }
-            }
+        entitySearchService.appendRunId(opContext, results);
 
             // TODO: We don't actually use this return value anywhere. Maybe we should just stop returning it altogether?
-            return "success";
+            return RESTLI_SUCCESS;
           } catch (ValidationException e) {
             throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, e.getMessage());
           }
@@ -385,14 +394,4 @@ public Task<String> restoreIndices(
         },
         MetricRegistry.name(this.getClass(), "restoreIndices"));
   }
-
-  private static void tryIndexRunId(
-          @Nonnull final OperationContext opContext,
-      final Urn urn,
-      final @Nullable SystemMetadata systemMetadata,
-      final EntitySearchService entitySearchService) {
-    if (systemMetadata != null && systemMetadata.hasRunId()) {
-      entitySearchService.appendRunId(opContext, urn.getEntityType(), urn, systemMetadata.getRunId());
-    }
-  }
 }
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java
index 9755a76848adf..0c374c29cf958 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java
@@ -12,6 +12,7 @@
 import static com.linkedin.metadata.entity.validation.ValidationUtils.*;
 import static com.linkedin.metadata.resources.restli.RestliConstants.*;
 import static com.linkedin.metadata.search.utils.SearchUtils.*;
+import static com.linkedin.metadata.utils.CriterionUtils.validateAndConvert;
 import static com.linkedin.metadata.utils.PegasusUtils.*;
 import static com.linkedin.metadata.utils.SystemMetadataUtils.generateSystemMetadataIfEmpty;
 
@@ -273,7 +274,7 @@ public Task<Void> ingest(
     String actorUrnStr = authentication.getActor().toUrnStr();
     final Urn urn = com.datahub.util.ModelUtils.getUrnFromSnapshotUnion(entity.getValue());
     final OperationContext opContext = OperationContext.asSession(
-            systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), getContext(),
+            systemOperationContext, RequestContext.builder().buildRestli(actorUrnStr, getContext(),
                     ACTION_INGEST, urn.getEntityType()), authorizer, authentication, true);
 
     if (!isAPIAuthorizedEntityUrns(
@@ -281,7 +282,7 @@ public Task<Void> ingest(
             CREATE,
             List.of(urn))) {
       throw new RestLiServiceException(
-          HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entity " + urn);
+          HttpStatus.S_403_FORBIDDEN, "User " + actorUrnStr + " is unauthorized to edit entity " + urn);
     }
 
     try {
@@ -319,7 +320,7 @@ public Task<Void> batchIngest(
             .map(Entity::getValue)
             .map(com.datahub.util.ModelUtils::getUrnFromSnapshotUnion).collect(Collectors.toList());
     final OperationContext opContext = OperationContext.asSession(
-            systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(),
+            systemOperationContext, RequestContext.builder().buildRestli(actorUrnStr,
                     getContext(), ACTION_BATCH_INGEST, urns.stream().map(Urn::getEntityType).collect(Collectors.toList())),
             authorizer, authentication, true);
 
@@ -327,7 +328,7 @@ public Task<Void> batchIngest(
             opContext,
             CREATE, urns)) {
       throw new RestLiServiceException(
-          HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entities.");
+          HttpStatus.S_403_FORBIDDEN, "User " + actorUrnStr +  " is unauthorized to edit entities.");
     }
 
     for (Entity entity : entities) {
@@ -401,7 +402,7 @@ public Task<SearchResult> search(
           // This API is not used by the frontend for search bars so we default to structured
           result =
               entitySearchService.search(opContext,
-                  List.of(entityName), input, filter, sortCriterionList, start, count);
+                  List.of(entityName), input, validateAndConvert(filter), sortCriterionList, start, count);
 
           if (!isAPIAuthorizedResult(
                   opContext,
@@ -448,7 +449,7 @@ public Task<SearchResult> searchAcrossEntities(
     log.info("GET SEARCH RESULTS ACROSS ENTITIES for {} with query {}", entityList, input);
     return RestliUtils.toTask(
         () -> {
-          SearchResult result = searchService.searchAcrossEntities(opContext, entityList, input, filter, sortCriterionList, start, count);
+          SearchResult result = searchService.searchAcrossEntities(opContext, entityList, input, validateAndConvert(filter), sortCriterionList, start, count);
           if (!isAPIAuthorizedResult(
                   opContext,
                   result)) {
@@ -514,7 +515,7 @@ public Task<ScrollResult> scrollAcrossEntities(
                   opContext,
                   entityList,
                   input,
-                  filter,
+                  validateAndConvert(filter),
                   sortCriterionList,
                   scrollId,
                   keepAlive,
@@ -583,7 +584,7 @@ public Task<LineageSearchResult> searchAcrossLineage(
                   entityList,
                   input,
                   maxHops,
-                  filter,
+                  validateAndConvert(filter),
                   sortCriterionList,
                   start,
                   count),
@@ -648,7 +649,7 @@ public Task<LineageScrollResult> scrollAcrossLineage(
                     entityList,
                     input,
                     maxHops,
-                    filter,
+                    validateAndConvert(filter),
                     sortCriterionList,
                     scrollId,
                     keepAlive,
@@ -683,10 +684,11 @@ public Task<ListResult> list(
 
     List<SortCriterion> sortCriterionList = getSortCriteria(sortCriteria, sortCriterion);
 
-    log.info("GET LIST RESULTS for {} with filter {}", entityName, filter);
+    final Filter finalFilter = validateAndConvert(filter);
+    log.info("GET LIST RESULTS for {} with filter {}", entityName, finalFilter);
     return RestliUtils.toTask(
         () -> {
-            SearchResult result = entitySearchService.filter(opContext, entityName, filter, sortCriterionList, start, count);
+            SearchResult result = entitySearchService.filter(opContext, entityName, finalFilter, sortCriterionList, start, count);
           if (!AuthUtil.isAPIAuthorizedResult(
                   opContext,
                   result)) {
@@ -927,7 +929,7 @@ public Task<DeleteEntityResponse> deleteEntity(
                   opContext.getEntityRegistry(), urn.getEntityType());
           if (aspectName != null && !timeseriesAspectNames.contains(aspectName)) {
             throw new UnsupportedOperationException(
-                String.format("Not supported for non-timeseries aspect '{}'.", aspectName));
+                String.format("Not supported for non-timeseries aspect %s.", aspectName));
           }
           List<String> timeseriesAspectsToDelete =
               (aspectName == null) ? timeseriesAspectNames : ImmutableList.of(aspectName);
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java
index 20209ddf44d64..896d81d3cbecc 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java
@@ -64,7 +64,8 @@ public class EntityV2Resource extends CollectionResourceTaskTemplate<String, Ent
   @Nonnull
   @WithSpan
   public Task<EntityResponse> get(
-      @Nonnull String urnStr, @QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames)
+      @Nonnull String urnStr, @QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames,
+      @QueryParam(PARAM_ALWAYS_INCLUDE_KEY_ASPECT) @Optional @Nullable Boolean alwaysIncludeKeyAspect)
       throws URISyntaxException {
     log.debug("GET V2 {}", urnStr);
     final Urn urn = Urn.createFromString(urnStr);
@@ -90,7 +91,7 @@ public Task<EntityResponse> get(
                   ? opContext.getEntityAspectNames(entityName)
                   : new HashSet<>(Arrays.asList(aspectNames));
           try {
-            return _entityService.getEntityV2(opContext, entityName, urn, projectedAspects);
+            return _entityService.getEntityV2(opContext, entityName, urn, projectedAspects, alwaysIncludeKeyAspect == null || alwaysIncludeKeyAspect);
           } catch (Exception e) {
             throw new RuntimeException(
                 String.format(
@@ -106,7 +107,8 @@ public Task<EntityResponse> get(
   @WithSpan
   public Task<Map<Urn, EntityResponse>> batchGet(
       @Nonnull Set<String> urnStrs,
-      @QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames)
+      @QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames,
+      @QueryParam(PARAM_ALWAYS_INCLUDE_KEY_ASPECT) @Optional @Nullable Boolean alwaysIncludeKeyAspect)
       throws URISyntaxException {
     log.debug("BATCH GET V2 {}", urnStrs.toString());
     final Set<Urn> urns = new HashSet<>();
@@ -138,7 +140,7 @@ public Task<Map<Urn, EntityResponse>> batchGet(
                   ? opContext.getEntityAspectNames(entityName)
                   : new HashSet<>(Arrays.asList(aspectNames));
           try {
-            return _entityService.getEntitiesV2(opContext, entityName, urns, projectedAspects);
+            return _entityService.getEntitiesV2(opContext, entityName, urns, projectedAspects, alwaysIncludeKeyAspect == null || alwaysIncludeKeyAspect);
           } catch (Exception e) {
             throw new RuntimeException(
                 String.format(
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java
index ef79a404c2145..11df52ad66709 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java
@@ -1,5 +1,7 @@
 package com.linkedin.metadata.resources.restli;
 
+import javax.annotation.Nullable;
+
 public final class RestliConstants {
   private RestliConstants() {}
 
@@ -21,6 +23,7 @@ private RestliConstants() {}
   public static final String PARAM_INPUT = "input";
   public static final String PARAM_MAX_HOPS = "maxHops";
   public static final String PARAM_ASPECTS = "aspects";
+  public static final String PARAM_ALWAYS_INCLUDE_KEY_ASPECT = "alwaysIncludeKeyAspect";
   public static final String PARAM_FILTER = "filter";
   public static final String PARAM_GROUP = "group";
   public static final String PARAM_SORT = "sort";
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliUtils.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliUtils.java
index 185874fac1382..a2092405da3ff 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliUtils.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliUtils.java
@@ -8,6 +8,7 @@
 import com.linkedin.parseq.Task;
 import com.linkedin.restli.common.HttpStatus;
 import com.linkedin.restli.server.RestLiServiceException;
+import io.datahubproject.metadata.exception.ActorAccessException;
 import java.util.Optional;
 import java.util.function.Supplier;
 import javax.annotation.Nonnull;
@@ -38,6 +39,8 @@ public static <T> Task<T> toTask(@Nonnull Supplier<T> supplier) {
       if (throwable instanceof IllegalArgumentException
           || throwable.getCause() instanceof IllegalArgumentException) {
         finalException = badRequestException(throwable.getMessage());
+      } else if (throwable.getCause() instanceof ActorAccessException) {
+          finalException = forbidden(throwable.getCause().getMessage());
       } else if (throwable instanceof APIThrottleException) {
         finalException = apiThrottled(throwable.getMessage());
       } else if (throwable instanceof RestLiServiceException) {
@@ -109,4 +112,9 @@ public static RestLiServiceException invalidArgumentsException(@Nullable String
   public static RestLiServiceException apiThrottled(@Nullable String message) {
     return new RestLiServiceException(HttpStatus.S_429_TOO_MANY_REQUESTS, message);
   }
+
+  @Nonnull
+  public static RestLiServiceException forbidden(@Nullable String message) {
+    return new RestLiServiceException(HttpStatus.S_403_FORBIDDEN, message);
+  }
 }
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java
index a0c3d46095160..426eff20c9c6e 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java
@@ -104,9 +104,10 @@ public Task<Void> batchIngest(@ActionParam(PARAM_BUCKETS) @Nonnull UsageAggregat
         () -> {
 
           final Authentication auth = AuthenticationContext.getAuthentication();
+          String actorUrnStr = auth.getActor().toUrnStr();
           Set<Urn> urns = Arrays.stream(buckets).sequential().map(UsageAggregation::getResource).collect(Collectors.toSet());
           final OperationContext opContext = OperationContext.asSession(
-                  systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(),
+                  systemOperationContext, RequestContext.builder().buildRestli(actorUrnStr, getContext(),
                           ACTION_BATCH_INGEST, urns.stream().map(Urn::getEntityType).collect(Collectors.toList())), _authorizer,
                   auth, true);
 
@@ -115,7 +116,7 @@ public Task<Void> batchIngest(@ActionParam(PARAM_BUCKETS) @Nonnull UsageAggregat
                   UPDATE,
                   urns)) {
             throw new RestLiServiceException(
-                HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entities.");
+                HttpStatus.S_403_FORBIDDEN, "User " + actorUrnStr + " is unauthorized to edit entities.");
           }
 
           for (UsageAggregation agg : buckets) {
diff --git a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java
index 9872f45648d7b..037b5b81fd4df 100644
--- a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java
+++ b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java
@@ -25,8 +25,10 @@
 import com.linkedin.metadata.event.EventProducer;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.registry.EntityRegistry;
+import com.linkedin.metadata.search.EntitySearchService;
 import com.linkedin.metadata.service.UpdateIndicesService;
 import com.linkedin.metadata.utils.GenericRecordUtils;
+import com.linkedin.mxe.GenericAspect;
 import com.linkedin.mxe.MetadataChangeLog;
 import com.linkedin.mxe.MetadataChangeProposal;
 import java.net.URISyntaxException;
@@ -65,6 +67,7 @@ public void setup() {
     aspectResource.setEntityService(entityService);
     opContext = TestOperationContexts.systemContextNoSearchAuthorization();
     aspectResource.setSystemOperationContext(opContext);
+    aspectResource.setEntitySearchService(mock(EntitySearchService.class));
     entityRegistry = opContext.getEntityRegistry();
   }
 
@@ -97,7 +100,7 @@ public void testAsyncDefaultAspects() throws URISyntaxException {
             .recordTemplate(mcp.getAspect())
             .auditStamp(new AuditStamp())
             .metadataChangeProposal(mcp)
-            .build(opContext.getAspectRetrieverOpt().get());
+            .build(opContext.getAspectRetriever());
     when(aspectDao.runInTransactionWithRetry(any(), any(), anyInt()))
         .thenReturn(
             List.of(List.of(
@@ -136,4 +139,31 @@ public void testAsyncDefaultAspects() throws URISyntaxException {
         .produceMetadataChangeLog(eq(urn), any(AspectSpec.class), any(MetadataChangeLog.class));
     verifyNoMoreInteractions(producer);
   }
+
+  @Test
+  public void testNoValidateAsync() throws URISyntaxException {
+    OperationContext noValidateOpContext = TestOperationContexts.systemContextNoValidate();
+    aspectResource.setSystemOperationContext(noValidateOpContext);
+    reset(producer, aspectDao);
+    MetadataChangeProposal mcp = new MetadataChangeProposal();
+    mcp.setEntityType(DATASET_ENTITY_NAME);
+    Urn urn = new DatasetUrn(new DataPlatformUrn("platform"), "name", FabricType.PROD);
+    mcp.setEntityUrn(urn);
+    GenericAspect properties = GenericRecordUtils.serializeAspect(new DatasetProperties().setName("name"));
+    mcp.setAspect(GenericRecordUtils.serializeAspect(properties));
+    mcp.setAspectName("notAnAspect");
+    mcp.setChangeType(ChangeType.UPSERT);
+    mcp.setSystemMetadata(new SystemMetadata());
+
+    Authentication mockAuthentication = mock(Authentication.class);
+    AuthenticationContext.setAuthentication(mockAuthentication);
+    Actor actor = new Actor(ActorType.USER, "user");
+    when(mockAuthentication.getActor()).thenReturn(actor);
+    aspectResource.ingestProposal(mcp, "true");
+    verify(producer, times(1)).produceMetadataChangeProposal(urn, mcp);
+    verifyNoMoreInteractions(producer);
+    verifyNoMoreInteractions(aspectDao);
+    reset(producer, aspectDao);
+    aspectResource.setSystemOperationContext(opContext);
+  }
 }
diff --git a/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java b/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java
index 7ed183e975f3b..1aea837818fe8 100644
--- a/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java
+++ b/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java
@@ -15,6 +15,8 @@
 import com.linkedin.timeseries.TimeseriesIndexSizeResult;
 import io.datahubproject.metadata.context.OperationContext;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -67,6 +69,16 @@ public List<EnvelopedAspect> getAspectValues(
     return List.of();
   }
 
+  @Nonnull
+  @Override
+  public Map<Urn, Map<String, EnvelopedAspect>> getLatestTimeseriesAspectValues(
+      @Nonnull OperationContext opContext,
+      @Nonnull Set<Urn> urns,
+      @Nonnull Set<String> aspectNames,
+      @Nullable Map<String, Long> beforeTimeMillis) {
+    return Map.of();
+  }
+
   @Nonnull
   @Override
   public GenericTable getAggregatedStats(
diff --git a/metadata-service/schema-registry-api/build.gradle b/metadata-service/schema-registry-api/build.gradle
index 201e98beb490e..525eceda685be 100644
--- a/metadata-service/schema-registry-api/build.gradle
+++ b/metadata-service/schema-registry-api/build.gradle
@@ -3,6 +3,7 @@ plugins {
     id 'java-library'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
 
 dependencies {
 // Dependencies for open api
diff --git a/metadata-service/schema-registry-servlet/build.gradle b/metadata-service/schema-registry-servlet/build.gradle
index f19237c32cee7..94ca8fe03822e 100644
--- a/metadata-service/schema-registry-servlet/build.gradle
+++ b/metadata-service/schema-registry-servlet/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'java'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
     implementation project(':metadata-service:factories')
     implementation project(':metadata-service:schema-registry-api')
diff --git a/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTest.java b/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTest.java
index 664766f204e46..e8deed00672da 100644
--- a/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTest.java
+++ b/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTest.java
@@ -1,6 +1,7 @@
 package io.datahubproject.openapi.test;
 
 import static com.linkedin.metadata.Constants.*;
+import static com.linkedin.metadata.config.kafka.KafkaConfiguration.DEFAULT_EVENT_CONSUMER_NAME;
 import static org.testng.Assert.*;
 
 import com.linkedin.common.urn.Urn;
@@ -199,7 +200,7 @@ public void testPEConsumption()
   @KafkaListener(
       id = "test-mcp-consumer",
       topics = Topics.METADATA_CHANGE_PROPOSAL,
-      containerFactory = "kafkaEventConsumer",
+      containerFactory = DEFAULT_EVENT_CONSUMER_NAME,
       properties = {"auto.offset.reset:earliest"})
   public void receiveMCP(ConsumerRecord<String, GenericRecord> consumerRecord) {
 
@@ -216,7 +217,7 @@ public void receiveMCP(ConsumerRecord<String, GenericRecord> consumerRecord) {
   @KafkaListener(
       id = "test-mcl-consumer",
       topics = Topics.METADATA_CHANGE_LOG_VERSIONED,
-      containerFactory = "kafkaEventConsumer",
+      containerFactory = DEFAULT_EVENT_CONSUMER_NAME,
       properties = {"auto.offset.reset:earliest"})
   public void receiveMCL(ConsumerRecord<String, GenericRecord> consumerRecord) {
 
@@ -232,7 +233,7 @@ public void receiveMCL(ConsumerRecord<String, GenericRecord> consumerRecord) {
   @KafkaListener(
       id = "test-pe-consumer",
       topics = Topics.PLATFORM_EVENT,
-      containerFactory = "kafkaEventConsumer",
+      containerFactory = DEFAULT_EVENT_CONSUMER_NAME,
       properties = {"auto.offset.reset:earliest"})
   public void receivePE(ConsumerRecord<String, GenericRecord> consumerRecord) {
 
diff --git a/metadata-service/services/build.gradle b/metadata-service/services/build.gradle
index ea1ff32cb3838..0e84580cc04da 100644
--- a/metadata-service/services/build.gradle
+++ b/metadata-service/services/build.gradle
@@ -3,6 +3,8 @@ plugins {
     id 'java-library'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 configurations {
     enhance
 }
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java
index dbe1f45fae29e..43d5e61c47bac 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java
@@ -78,7 +78,15 @@ public enum DataHubUsageEventType {
   EMBED_PROFILE_VIEW_EVENT("EmbedProfileViewEvent"),
   EMBED_PROFILE_VIEW_IN_DATAHUB_EVENT("EmbedProfileViewInDataHubEvent"),
   EMBED_LOOKUP_NOT_FOUND_EVENT("EmbedLookupNotFoundEvent"),
-  CREATE_BUSINESS_ATTRIBUTE("CreateBusinessAttributeEvent");
+  CREATE_BUSINESS_ATTRIBUTE("CreateBusinessAttributeEvent"),
+  CREATE_STRUCTURED_PROPERTY_CLICK_EVENT("CreateStructuredPropertyClickEvent"),
+  CREATE_STRUCTURED_PROPERTY_EVENT("CreateStructuredPropertyEvent"),
+  EDIT_STRUCTURED_PROPERTY_EVENT("EditStructuredPropertyEvent"),
+  DELETE_STRUCTURED_PROPERTY_EVENT("DeleteStructuredPropertyEvent"),
+  VIEW_STRUCTURED_PROPERTY_EVENT("ViewStructuredPropertyEvent"),
+  APPLY_STRUCTURED_PROPERTY_EVENT("ApplyStructuredPropertyEvent"),
+  UPDATE_STRUCTURED_PROPERTY_ON_ASSET_EVENT("UpdateStructuredPropertyOnAssetEvent"),
+  REMOVE_STRUCTURED_PROPERTY_EVENT("RemoveStructuredPropertyEvent");
 
   private final String type;
 
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java
index a3c57a19eddd5..2259b9365ff64 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java
@@ -1,16 +1,11 @@
 package com.linkedin.metadata.entity;
 
 import static com.linkedin.metadata.search.utils.QueryUtils.*;
-import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion;
 
 import com.datahub.util.RecordUtils;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.linkedin.common.AuditStamp;
-import com.linkedin.common.FormAssociation;
-import com.linkedin.common.FormAssociationArray;
-import com.linkedin.common.FormVerificationAssociation;
-import com.linkedin.common.FormVerificationAssociationArray;
 import com.linkedin.common.Forms;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
@@ -20,6 +15,7 @@
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.form.FormInfo;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.aspect.models.graph.RelatedEntity;
 import com.linkedin.metadata.graph.GraphService;
@@ -28,10 +24,6 @@
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.models.RelationshipFieldSpec;
 import com.linkedin.metadata.models.extractor.FieldExtractor;
-import com.linkedin.metadata.query.filter.Condition;
-import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
-import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
-import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.query.filter.RelationshipDirection;
 import com.linkedin.metadata.run.DeleteReferencesResponse;
@@ -568,65 +560,44 @@ private AssetScrollResult getAssetsReferencingUrn(
     result.assets = new ArrayList<>();
 
     if (deletedUrn.getEntityType().equals("form")) {
-      // first, get all entities with this form assigned on it
-      final CriterionArray incompleteFormsArray = new CriterionArray();
-      incompleteFormsArray.add(
-          buildCriterion("incompleteForms", Condition.EQUAL, deletedUrn.toString()));
-      final CriterionArray completedFormsArray = new CriterionArray();
-      completedFormsArray.add(
-          buildCriterion("completedForms", Condition.EQUAL, deletedUrn.toString()));
-      // next, get all metadata tests created for this form
-      final CriterionArray metadataTestSourceArray = new CriterionArray();
-      metadataTestSourceArray.add(
-          buildCriterion("sourceEntity", Condition.EQUAL, deletedUrn.toString()));
-      metadataTestSourceArray.add(buildCriterion("sourceType", Condition.EQUAL, "FORMS"));
-      Filter filter =
-          new Filter()
-              .setOr(
-                  new ConjunctiveCriterionArray(
-                      new ConjunctiveCriterion().setAnd(incompleteFormsArray),
-                      new ConjunctiveCriterion().setAnd(completedFormsArray),
-                      new ConjunctiveCriterion().setAnd(metadataTestSourceArray)));
-      ScrollResult scrollResult =
-          _searchService.structuredScroll(
-              opContext,
-              ImmutableList.of(
-                  "dataset",
-                  "dataJob",
-                  "dataFlow",
-                  "chart",
-                  "dashboard",
-                  "corpuser",
-                  "corpGroup",
-                  "domain",
-                  "container",
-                  "glossaryTerm",
-                  "glossaryNode",
-                  "mlModel",
-                  "mlModelGroup",
-                  "mlFeatureTable",
-                  "mlFeature",
-                  "mlPrimaryKey",
-                  "schemaField",
-                  "dataProduct",
-                  "test"),
-              "*",
-              filter,
-              null,
-              scrollId,
-              "5m",
-              dryRun ? 1 : BATCH_SIZE); // need to pass in 1 for count otherwise get index error
-      if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) {
-        return result;
-      }
-      result.scrollId = scrollResult.getScrollId();
-      result.totalAssetCount = scrollResult.getNumEntities();
-      result.assets =
-          scrollResult.getEntities().stream()
-              .map(SearchEntity::getEntity)
-              .collect(Collectors.toList());
+      Filter filter = DeleteEntityUtils.getFilterForFormDeletion(deletedUrn);
+      List<String> entityNames = DeleteEntityUtils.getEntityNamesForFormDeletion();
+      return scrollForAssets(opContext, result, filter, entityNames, scrollId, dryRun);
+    }
+    if (deletedUrn.getEntityType().equals("structuredProperty")) {
+      Filter filter = DeleteEntityUtils.getFilterForStructuredPropertyDeletion(deletedUrn);
+      List<String> entityNames = DeleteEntityUtils.getEntityNamesForStructuredPropertyDeletion();
+      return scrollForAssets(opContext, result, filter, entityNames, scrollId, dryRun);
+    }
+    return result;
+  }
+
+  private AssetScrollResult scrollForAssets(
+      @Nonnull OperationContext opContext,
+      AssetScrollResult result,
+      final @Nullable Filter filter,
+      final @Nonnull List<String> entityNames,
+      @Nullable String scrollId,
+      final boolean dryRun) {
+    ScrollResult scrollResult =
+        _searchService.structuredScroll(
+            opContext,
+            entityNames,
+            "*",
+            filter,
+            null,
+            scrollId,
+            "5m",
+            dryRun ? 1 : BATCH_SIZE); // need to pass in 1 for count otherwise get index error
+    if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) {
       return result;
     }
+    result.scrollId = scrollResult.getScrollId();
+    result.totalAssetCount = scrollResult.getNumEntities();
+    result.assets =
+        scrollResult.getEntities().stream()
+            .map(SearchEntity::getEntity)
+            .collect(Collectors.toList());
     return result;
   }
 
@@ -641,7 +612,7 @@ private List<MetadataChangeProposal> deleteSearchReferencesForAsset(
     }
 
     List<MetadataChangeProposal> mcps = new ArrayList<>();
-    List<String> aspectsToUpdate = getAspectsToUpdate(deletedUrn);
+    List<String> aspectsToUpdate = getAspectsToUpdate(deletedUrn, assetUrn);
     aspectsToUpdate.forEach(
         aspectName -> {
           try {
@@ -667,10 +638,15 @@ private List<MetadataChangeProposal> deleteSearchReferencesForAsset(
    * <p>TODO: extend this to support other types of deletes and be more dynamic depending on aspects
    * that the asset has
    */
-  private List<String> getAspectsToUpdate(@Nonnull final Urn deletedUrn) {
+  private List<String> getAspectsToUpdate(
+      @Nonnull final Urn deletedUrn, @Nonnull final Urn assetUrn) {
     if (deletedUrn.getEntityType().equals("form")) {
       return ImmutableList.of("forms");
     }
+    if (deletedUrn.getEntityType().equals("structuredProperty")
+        && assetUrn.getEntityType().equals("form")) {
+      return ImmutableList.of("formInfo");
+    }
     return new ArrayList<>();
   }
 
@@ -697,6 +673,9 @@ private MetadataChangeProposal updateAspectForSearchReference(
     if (aspectName.equals("forms")) {
       return updateFormsAspect(opContext, assetUrn, deletedUrn);
     }
+    if (aspectName.equals("formInfo") && deletedUrn.getEntityType().equals("structuredProperty")) {
+      return updateFormInfoAspect(opContext, assetUrn, deletedUrn);
+    }
     return null;
   }
 
@@ -708,35 +687,20 @@ private MetadataChangeProposal updateFormsAspect(
     RecordTemplate record = _entityService.getLatestAspect(opContext, assetUrn, "forms");
     if (record == null) return null;
 
-    Forms formsAspect = new Forms(record.data());
-    final AtomicReference<Forms> updatedAspect;
-    try {
-      updatedAspect = new AtomicReference<>(formsAspect.copy());
-    } catch (Exception e) {
-      throw new RuntimeException("Failed to copy the forms aspect for updating", e);
-    }
-
-    List<FormAssociation> incompleteForms =
-        formsAspect.getIncompleteForms().stream()
-            .filter(incompleteForm -> !incompleteForm.getUrn().equals(deletedUrn))
-            .collect(Collectors.toList());
-    List<FormAssociation> completedForms =
-        formsAspect.getCompletedForms().stream()
-            .filter(completedForm -> !completedForm.getUrn().equals(deletedUrn))
-            .collect(Collectors.toList());
-    final List<FormVerificationAssociation> verifications =
-        formsAspect.getVerifications().stream()
-            .filter(verification -> !verification.getForm().equals(deletedUrn))
-            .collect(Collectors.toList());
+    return DeleteEntityUtils.removeFormFromFormsAspect(
+        new Forms(record.data()), assetUrn, deletedUrn);
+  }
 
-    updatedAspect.get().setIncompleteForms(new FormAssociationArray(incompleteForms));
-    updatedAspect.get().setCompletedForms(new FormAssociationArray(completedForms));
-    updatedAspect.get().setVerifications(new FormVerificationAssociationArray(verifications));
+  @Nullable
+  private MetadataChangeProposal updateFormInfoAspect(
+      @Nonnull OperationContext opContext,
+      @Nonnull final Urn assetUrn,
+      @Nonnull final Urn deletedUrn) {
+    RecordTemplate record = _entityService.getLatestAspect(opContext, assetUrn, "formInfo");
+    if (record == null) return null;
 
-    if (!formsAspect.equals(updatedAspect.get())) {
-      return AspectUtils.buildMetadataChangeProposal(assetUrn, "forms", updatedAspect.get());
-    }
-    return null;
+    return DeleteEntityUtils.createFormInfoUpdateProposal(
+        new FormInfo(record.data()), assetUrn, deletedUrn);
   }
 
   private AuditStamp createAuditStamp() {
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java
index 0a8b5880e5bce..20dc104e1b436 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java
@@ -1,5 +1,14 @@
 package com.linkedin.metadata.entity;
 
+import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion;
+
+import com.google.common.collect.ImmutableList;
+import com.linkedin.common.FormAssociation;
+import com.linkedin.common.FormAssociationArray;
+import com.linkedin.common.FormVerificationAssociation;
+import com.linkedin.common.FormVerificationAssociationArray;
+import com.linkedin.common.Forms;
+import com.linkedin.common.urn.Urn;
 import com.linkedin.data.DataComplex;
 import com.linkedin.data.DataList;
 import com.linkedin.data.DataMap;
@@ -9,8 +18,22 @@
 import com.linkedin.data.schema.RecordDataSchema;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.entity.Aspect;
+import com.linkedin.form.FormInfo;
+import com.linkedin.form.FormPrompt;
+import com.linkedin.form.FormPromptArray;
+import com.linkedin.metadata.query.filter.Condition;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.CriterionArray;
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.utils.CriterionUtils;
+import com.linkedin.mxe.MetadataChangeProposal;
 import java.util.List;
 import java.util.ListIterator;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -207,4 +230,147 @@ private static DataComplex removeValueFromArray(
     }
     return aspectList;
   }
+
+  /*
+   * Form Deletion Section
+   */
+
+  // We need to update assets that have this form on them in one way or another
+  public static Filter getFilterForFormDeletion(@Nonnull final Urn deletedUrn) {
+    // first, get all entities with this form assigned on it
+    final CriterionArray incompleteFormsArray = new CriterionArray();
+    incompleteFormsArray.add(
+        buildCriterion("incompleteForms", Condition.EQUAL, deletedUrn.toString()));
+    final CriterionArray completedFormsArray = new CriterionArray();
+    completedFormsArray.add(
+        buildCriterion("completedForms", Condition.EQUAL, deletedUrn.toString()));
+    // next, get all metadata tests created for this form
+    final CriterionArray metadataTestSourceArray = new CriterionArray();
+    metadataTestSourceArray.add(
+        buildCriterion("sourceEntity", Condition.EQUAL, deletedUrn.toString()));
+    metadataTestSourceArray.add(buildCriterion("sourceType", Condition.EQUAL, "FORMS"));
+    return new Filter()
+        .setOr(
+            new ConjunctiveCriterionArray(
+                new ConjunctiveCriterion().setAnd(incompleteFormsArray),
+                new ConjunctiveCriterion().setAnd(completedFormsArray),
+                new ConjunctiveCriterion().setAnd(metadataTestSourceArray)));
+  }
+
+  @Nullable
+  public static MetadataChangeProposal removeFormFromFormsAspect(
+      @Nonnull Forms formsAspect, @Nonnull final Urn assetUrn, @Nonnull final Urn deletedUrn) {
+    final AtomicReference<Forms> updatedAspect;
+    try {
+      updatedAspect = new AtomicReference<>(formsAspect.copy());
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to copy the forms aspect for updating", e);
+    }
+
+    List<FormAssociation> incompleteForms =
+        formsAspect.getIncompleteForms().stream()
+            .filter(incompleteForm -> !incompleteForm.getUrn().equals(deletedUrn))
+            .collect(Collectors.toList());
+    List<FormAssociation> completedForms =
+        formsAspect.getCompletedForms().stream()
+            .filter(completedForm -> !completedForm.getUrn().equals(deletedUrn))
+            .collect(Collectors.toList());
+    final List<FormVerificationAssociation> verifications =
+        formsAspect.getVerifications().stream()
+            .filter(verification -> !verification.getForm().equals(deletedUrn))
+            .collect(Collectors.toList());
+
+    updatedAspect.get().setIncompleteForms(new FormAssociationArray(incompleteForms));
+    updatedAspect.get().setCompletedForms(new FormAssociationArray(completedForms));
+    updatedAspect.get().setVerifications(new FormVerificationAssociationArray(verifications));
+
+    if (!formsAspect.equals(updatedAspect.get())) {
+      return AspectUtils.buildMetadataChangeProposal(assetUrn, "forms", updatedAspect.get());
+    }
+    return null;
+  }
+
+  // all assets that could have a form associated with them
+  public static List<String> getEntityNamesForFormDeletion() {
+    return ImmutableList.of(
+        "dataset",
+        "dataJob",
+        "dataFlow",
+        "chart",
+        "dashboard",
+        "corpuser",
+        "corpGroup",
+        "domain",
+        "container",
+        "glossaryTerm",
+        "glossaryNode",
+        "mlModel",
+        "mlModelGroup",
+        "mlFeatureTable",
+        "mlFeature",
+        "mlPrimaryKey",
+        "schemaField",
+        "dataProduct",
+        "test");
+  }
+
+  /*
+   * Structured Property Deletion Section
+   */
+
+  // get forms that have this structured property referenced in a prompt
+  public static Filter getFilterForStructuredPropertyDeletion(@Nonnull final Urn deletedUrn) {
+    final CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(
+        CriterionUtils.buildCriterion(
+            "structuredPropertyPromptUrns", Condition.EQUAL, deletedUrn.toString()));
+    return new Filter()
+        .setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(criterionArray)));
+  }
+
+  // only need to update forms manually when deleting structured props
+  public static List<String> getEntityNamesForStructuredPropertyDeletion() {
+    return ImmutableList.of("form");
+  }
+
+  @Nullable
+  public static MetadataChangeProposal createFormInfoUpdateProposal(
+      @Nonnull FormInfo formsAspect, @Nonnull final Urn assetUrn, @Nonnull final Urn deletedUrn) {
+    final FormInfo updatedFormInfo = removePromptsFromFormInfoAspect(formsAspect, deletedUrn);
+
+    if (!formsAspect.equals(updatedFormInfo)) {
+      return AspectUtils.buildMetadataChangeProposal(assetUrn, "formInfo", updatedFormInfo);
+    }
+
+    return null;
+  }
+
+  // remove any prompts referencing the deleted structured property urn
+  @Nonnull
+  public static FormInfo removePromptsFromFormInfoAspect(
+      @Nonnull FormInfo formsAspect, @Nonnull final Urn deletedUrn) {
+    final AtomicReference<FormInfo> updatedAspect;
+    try {
+      updatedAspect = new AtomicReference<>(formsAspect.copy());
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to copy the formInfo aspect for updating", e);
+    }
+
+    // filter out any prompt that has this structured property referenced on it
+    List<FormPrompt> filteredPrompts =
+        formsAspect.getPrompts().stream()
+            .filter(
+                prompt -> {
+                  if (prompt.getStructuredPropertyParams() != null
+                      && prompt.getStructuredPropertyParams().getUrn() != null) {
+                    return !prompt.getStructuredPropertyParams().getUrn().equals(deletedUrn);
+                  }
+                  return true;
+                })
+            .collect(Collectors.toList());
+
+    updatedAspect.get().setPrompts(new FormPromptArray(filteredPrompts));
+
+    return updatedAspect.get();
+  }
 }
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java
index 66f7ff50a3624..57af4aa05fff6 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java
@@ -41,13 +41,15 @@ public interface EntityService<U extends ChangeMCP> {
    * @param urns urns for the entities
    * @param aspectName aspect for the entity, if null, assumes key aspect
    * @param includeSoftDelete including soft deleted entities
+   * @param forUpdate whether the operation is intending to write to this row in a tx
    * @return set of urns with the specified aspect existing
    */
   Set<Urn> exists(
       @Nonnull OperationContext opContext,
       @Nonnull final Collection<Urn> urns,
       @Nullable String aspectName,
-      boolean includeSoftDelete);
+      boolean includeSoftDelete,
+      boolean forUpdate);
 
   /**
    * Just whether the entity/aspect exists, prefer batched method.
@@ -62,20 +64,37 @@ default boolean exists(
       @Nonnull Urn urn,
       @Nullable String aspectName,
       boolean includeSoftDelete) {
-    return exists(opContext, Set.of(urn), aspectName, includeSoftDelete).contains(urn);
+    return exists(opContext, Set.of(urn), aspectName, includeSoftDelete, false).contains(urn);
   }
 
   /**
    * Returns a set of urns of entities that exist (has materialized aspects).
    *
    * @param urns the list of urns of the entities to check
+   * @param includeSoftDelete including soft deleted entities
    * @return a set of urns of entities that exist.
    */
   default Set<Urn> exists(
       @Nonnull OperationContext opContext,
       @Nonnull final Collection<Urn> urns,
       boolean includeSoftDelete) {
-    return exists(opContext, urns, null, includeSoftDelete);
+    return exists(opContext, urns, null, includeSoftDelete, false);
+  }
+
+  /**
+   * Returns a set of urns of entities that exist (has materialized aspects).
+   *
+   * @param urns the list of urns of the entities to check
+   * @param includeSoftDelete including soft deleted entities
+   * @param forUpdate whether the operation is intending to write to this row in a tx
+   * @return a set of urns of entities that exist.
+   */
+  default Set<Urn> exists(
+      @Nonnull OperationContext opContext,
+      @Nonnull final Collection<Urn> urns,
+      boolean includeSoftDelete,
+      boolean forUpdate) {
+    return exists(opContext, urns, null, includeSoftDelete, forUpdate);
   }
 
   /**
@@ -86,18 +105,33 @@ default Set<Urn> exists(
    */
   default Set<Urn> exists(
       @Nonnull OperationContext opContext, @Nonnull final Collection<Urn> urns) {
-    return exists(opContext, urns, true);
+    return exists(opContext, urns, true, false);
   }
 
   /**
    * Returns whether the urn of the entity exists (has materialized aspects).
    *
    * @param urn the urn of the entity to check
+   * @param includeSoftDelete including soft deleted entities
    * @return entities exists.
    */
   default boolean exists(
       @Nonnull OperationContext opContext, @Nonnull Urn urn, boolean includeSoftDelete) {
-    return exists(opContext, List.of(urn), includeSoftDelete).contains(urn);
+    return exists(opContext, List.of(urn), includeSoftDelete, false).contains(urn);
+  }
+
+  /**
+   * Returns whether the urn of the entity exists (has materialized aspects).
+   *
+   * @param urn the urn of the entity to check
+   * @return entities exists.
+   */
+  default boolean exists(
+      @Nonnull OperationContext opContext,
+      @Nonnull Urn urn,
+      boolean includeSoftDelete,
+      boolean forUpdate) {
+    return exists(opContext, List.of(urn), includeSoftDelete, forUpdate).contains(urn);
   }
 
   /**
@@ -107,7 +141,7 @@ default boolean exists(
    * @return entities exists.
    */
   default boolean exists(@Nonnull OperationContext opContext, @Nonnull Urn urn) {
-    return exists(opContext, urn, true);
+    return exists(opContext, urn, true, false);
   }
 
   /**
@@ -137,7 +171,8 @@ default Map<Urn, List<RecordTemplate>> getLatestAspects(
   Map<String, RecordTemplate> getLatestAspectsForUrn(
       @Nonnull OperationContext opContext,
       @Nonnull final Urn urn,
-      @Nonnull final Set<String> aspectNames);
+      @Nonnull final Set<String> aspectNames,
+      boolean forUpdate);
 
   /**
    * Retrieves an aspect having a specific {@link Urn}, name, & version.
@@ -363,7 +398,9 @@ List<UpdateAspectResult> ingestAspects(
    * @param auditStamp an {@link AuditStamp} containing metadata about the writer & current time
    * @param systemMetadata
    * @return the {@link RecordTemplate} representation of the written aspect object
+   * @deprecated See Conditional Write ChangeType CREATE
    */
+  @Deprecated
   RecordTemplate ingestAspectIfNotPresent(
       @Nonnull OperationContext opContext,
       @Nonnull Urn urn,
@@ -487,7 +524,7 @@ RollbackRunResult rollbackWithConditions(
       Map<String, String> conditions,
       boolean hardDelete);
 
-  Set<IngestResult> ingestProposal(
+  List<IngestResult> ingestProposal(
       @Nonnull OperationContext opContext, AspectsBatch aspectsBatch, final boolean async);
 
   /**
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestResult.java
index d3f8b507bb14a..f8b76db110c08 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestResult.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestResult.java
@@ -2,6 +2,7 @@
 
 import com.linkedin.common.urn.Urn;
 import com.linkedin.metadata.aspect.batch.BatchItem;
+import javax.annotation.Nullable;
 import lombok.Builder;
 import lombok.Value;
 
@@ -10,6 +11,7 @@
 public class IngestResult {
   Urn urn;
   BatchItem request;
+  @Nullable UpdateAspectResult result;
   boolean publishedMCL;
   boolean processedMCL;
   boolean publishedMCP;
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
index 26f2335d0c59f..da9ba8c684f61 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
@@ -1,15 +1,21 @@
 package com.linkedin.metadata.search;
 
 import com.linkedin.common.urn.Urn;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.browse.BrowseResult;
 import com.linkedin.metadata.browse.BrowseResultV2;
+import com.linkedin.metadata.entity.IngestResult;
 import com.linkedin.metadata.query.AutoCompleteResult;
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.query.filter.SortCriterion;
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
+import com.linkedin.util.Pair;
 import io.datahubproject.metadata.context.OperationContext;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.opensearch.action.explain.ExplainResponse;
@@ -59,15 +65,10 @@ void deleteDocument(
   /**
    * Appends a run id to the list for a certain document
    *
-   * @param entityName name of the entity
    * @param urn the urn of the user
    * @param runId the ID of the run
    */
-  void appendRunId(
-      @Nonnull OperationContext opContext,
-      @Nonnull String entityName,
-      @Nonnull Urn urn,
-      @Nullable String runId);
+  void appendRunId(@Nonnull OperationContext opContext, @Nonnull Urn urn, @Nullable String runId);
 
   /**
    * Gets a list of documents that match given search request. The results are aggregated and
@@ -329,4 +330,41 @@ ExplainResponse explain(
    * @return convent
    */
   IndexConvention getIndexConvention();
+
+  default void appendRunId(
+      @Nonnull final OperationContext opContext, @Nonnull List<IngestResult> results) {
+
+    // Only updates with runId
+    Map<Pair<Urn, String>, Set<BatchItem>> urnRunIdToBatchItem =
+        results.stream()
+            .filter(Objects::nonNull)
+            .filter(
+                result -> result.getUrn() != null && (result.isProcessedMCL() || result.isUpdate()))
+            .filter(
+                result ->
+                    result.getRequest() != null
+                        && result.getRequest().getSystemMetadata() != null
+                        && result.getRequest().getSystemMetadata().hasRunId())
+            .map(
+                result ->
+                    Map.entry(
+                        Pair.of(
+                            result.getUrn(), result.getRequest().getSystemMetadata().getRunId()),
+                        result))
+            .collect(
+                Collectors.groupingBy(
+                    Map.Entry::getKey,
+                    Collectors.mapping(e -> e.getValue().getRequest(), Collectors.toSet())));
+
+    // Only update if not key aspect (document doesn't exist)
+    urnRunIdToBatchItem.entrySet().stream()
+        .filter(
+            entry ->
+                entry.getValue().stream()
+                    .noneMatch(
+                        item ->
+                            item.getEntitySpec().getKeyAspectName().equals(item.getAspectName())))
+        .forEach(
+            entry -> appendRunId(opContext, entry.getKey().getKey(), entry.getKey().getValue()));
+  }
 }
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java
index 68c82f0ef2e0d..b10debb7a61fc 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java
@@ -12,6 +12,8 @@
 import com.linkedin.timeseries.TimeseriesIndexSizeResult;
 import io.datahubproject.metadata.context.OperationContext;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -116,6 +118,22 @@ List<EnvelopedAspect> getAspectValues(
       @Nullable final Filter filter,
       @Nullable final SortCriterion sort);
 
+  /**
+   * Returns the latest value for the given URNs and aspects
+   *
+   * @param opContext operation context
+   * @param urns the urns
+   * @param aspectNames the aspects
+   * @param endTimeMillis fetch latest aspect before this time in milliseconds for each aspect
+   * @return Map of the urns
+   */
+  @Nonnull
+  Map<Urn, Map<String, EnvelopedAspect>> getLatestTimeseriesAspectValues(
+      @Nonnull OperationContext opContext,
+      @Nonnull final Set<Urn> urns,
+      @Nonnull final Set<String> aspectNames,
+      @Nullable final Map<String, Long> endTimeMillis);
+
   /**
    * Perform a arbitrary aggregation query over a set of Time-Series aspects. This is used to answer
    * arbitrary questions about the Time-Series aspects that we have.
diff --git a/metadata-service/services/src/test/java/com/linkedin/metadata/service/search/EntitySearchServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/search/EntitySearchServiceTest.java
new file mode 100644
index 0000000000000..41e2c2f006e94
--- /dev/null
+++ b/metadata-service/services/src/test/java/com/linkedin/metadata/service/search/EntitySearchServiceTest.java
@@ -0,0 +1,345 @@
+package com.linkedin.metadata.service.search;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.metadata.aspect.batch.BatchItem;
+import com.linkedin.metadata.browse.BrowseResult;
+import com.linkedin.metadata.browse.BrowseResultV2;
+import com.linkedin.metadata.entity.IngestResult;
+import com.linkedin.metadata.entity.UpdateAspectResult;
+import com.linkedin.metadata.query.AutoCompleteResult;
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.search.EntitySearchService;
+import com.linkedin.metadata.search.ScrollResult;
+import com.linkedin.metadata.search.SearchResult;
+import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
+import com.linkedin.mxe.SystemMetadata;
+import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.opensearch.action.explain.ExplainResponse;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+public class EntitySearchServiceTest {
+  private static final Urn TEST_URN =
+      UrnUtils.getUrn(
+          "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.human_profiles,PROD)");
+
+  private OperationContext opContext = TestOperationContexts.systemContextNoValidate();
+
+  private EntitySearchService testInstance;
+
+  @BeforeClass
+  public void setup() {
+    testInstance = spy(new TestEntitySearchService());
+  }
+
+  @Test
+  public void testAppendRunId_EmptyList() {
+    List<IngestResult> results = new ArrayList<>();
+    testInstance.appendRunId(opContext, results);
+    // Verify no interactions since list is empty
+    verify(testInstance, never()).appendRunId(any(), any(Urn.class), anyString());
+  }
+
+  @Test
+  public void testAppendRunId_NullResults() {
+    List<IngestResult> results = new ArrayList<>();
+    results.add(null);
+    testInstance.appendRunId(opContext, results);
+    // Verify no interactions since all results are null
+    verify(testInstance, never()).appendRunId(any(), any(Urn.class), anyString());
+  }
+
+  @Test
+  public void testAppendRunId_ValidResult() {
+    // Create test data
+    List<IngestResult> results = new ArrayList<>();
+    IngestResult result = mock(IngestResult.class);
+    BatchItem mockRequest = mock(BatchItem.class);
+    SystemMetadata mockSystemMetadata = mock(SystemMetadata.class);
+
+    // Setup mock behaviors
+    when(result.getUrn()).thenReturn(TEST_URN);
+    when(result.isProcessedMCL()).thenReturn(true);
+    when(result.getResult()).thenReturn(mock(UpdateAspectResult.class));
+    when(result.getRequest()).thenReturn(mockRequest);
+    when(mockRequest.getSystemMetadata()).thenReturn(mockSystemMetadata);
+    when(mockSystemMetadata.hasRunId()).thenReturn(true);
+    when(mockSystemMetadata.getRunId()).thenReturn("test-run-id");
+    when(mockRequest.getEntitySpec())
+        .thenReturn(opContext.getEntityRegistry().getEntitySpec(TEST_URN.getEntityType()));
+    when(mockRequest.getAspectName()).thenReturn("status");
+
+    results.add(result);
+
+    // Execute
+    testInstance.appendRunId(opContext, results);
+
+    // Verify appendRunId was called with correct parameters
+    verify(testInstance).appendRunId(eq(opContext), eq(TEST_URN), eq("test-run-id"));
+  }
+
+  @Test
+  public void testAppendRunId_KeyAspectMatch() {
+    // Create test data
+    List<IngestResult> results = new ArrayList<>();
+    IngestResult result = mock(IngestResult.class);
+    BatchItem mockRequest = mock(BatchItem.class);
+    SystemMetadata mockSystemMetadata = mock(SystemMetadata.class);
+
+    // Setup mock behaviors
+    when(result.getUrn()).thenReturn(TEST_URN);
+    when(result.isProcessedMCL()).thenReturn(true);
+    when(result.getResult()).thenReturn(mock(UpdateAspectResult.class));
+    when(result.getRequest()).thenReturn(mockRequest);
+    when(mockRequest.getSystemMetadata()).thenReturn(mockSystemMetadata);
+    when(mockSystemMetadata.hasRunId()).thenReturn(true);
+    when(mockSystemMetadata.getRunId()).thenReturn("test-run-id");
+    when(mockRequest.getEntitySpec())
+        .thenReturn(opContext.getEntityRegistry().getEntitySpec(TEST_URN.getEntityType()));
+    when(mockRequest.getAspectName())
+        .thenReturn(
+            opContext
+                .getEntityRegistry()
+                .getEntitySpec(TEST_URN.getEntityType())
+                .getKeyAspectName());
+
+    results.add(result);
+
+    // Execute
+    testInstance.appendRunId(opContext, results);
+
+    // Verify appendRunId was not called because aspect names match
+    verify(testInstance, never()).appendRunId(any(), any(Urn.class), anyString());
+  }
+
+  @Test
+  public void testAppendRunId_NoRunId() {
+    // Create test data
+    List<IngestResult> results = new ArrayList<>();
+    IngestResult result = mock(IngestResult.class);
+    BatchItem mockRequest = mock(BatchItem.class);
+    SystemMetadata mockSystemMetadata = mock(SystemMetadata.class);
+
+    // Setup mock behaviors
+    when(result.getUrn()).thenReturn(TEST_URN);
+    when(result.isProcessedMCL()).thenReturn(true);
+    when(result.getResult()).thenReturn(mock(UpdateAspectResult.class));
+    when(result.getRequest()).thenReturn(mockRequest);
+    when(mockRequest.getSystemMetadata()).thenReturn(mockSystemMetadata);
+    when(mockSystemMetadata.hasRunId()).thenReturn(false);
+
+    results.add(result);
+
+    // Execute
+    testInstance.appendRunId(opContext, results);
+
+    // Verify appendRunId was not called because there's no run ID
+    verify(testInstance, never()).appendRunId(any(), any(Urn.class), anyString());
+  }
+
+  @Test
+  public void testAppendRunId_NotProcessedOrUpdated() {
+    // Create test data
+    List<IngestResult> results = new ArrayList<>();
+    IngestResult result = mock(IngestResult.class);
+
+    // Setup mock behaviors
+    when(result.getUrn()).thenReturn(TEST_URN);
+    when(result.isProcessedMCL()).thenReturn(false);
+    when(result.isUpdate()).thenReturn(false);
+
+    results.add(result);
+
+    // Execute
+    testInstance.appendRunId(opContext, results);
+
+    // Verify appendRunId was not called because result is neither processed nor updated
+    verify(testInstance, never()).appendRunId(any(), any(Urn.class), anyString());
+  }
+
+  private static class TestEntitySearchService implements EntitySearchService {
+    @Override
+    public void clear(OperationContext opContext) {}
+
+    @Override
+    public long docCount(OperationContext opContext, String entityName, Filter filter) {
+      return 0;
+    }
+
+    @Override
+    public void upsertDocument(
+        OperationContext opContext, String entityName, String document, String docId) {}
+
+    @Override
+    public void deleteDocument(OperationContext opContext, String entityName, String docId) {}
+
+    @Override
+    public void appendRunId(OperationContext opContext, Urn urn, String runId) {}
+
+    @Override
+    public SearchResult search(
+        OperationContext opContext,
+        List<String> entityNames,
+        String input,
+        Filter postFilters,
+        List<SortCriterion> sortCriteria,
+        int from,
+        int size) {
+      return null;
+    }
+
+    @Override
+    public SearchResult search(
+        OperationContext opContext,
+        List<String> entityNames,
+        String input,
+        Filter postFilters,
+        List<SortCriterion> sortCriteria,
+        int from,
+        int size,
+        List<String> facets) {
+      return null;
+    }
+
+    @Override
+    public SearchResult filter(
+        OperationContext opContext,
+        String entityName,
+        Filter filters,
+        List<SortCriterion> sortCriteria,
+        int from,
+        int size) {
+      return null;
+    }
+
+    @Override
+    public AutoCompleteResult autoComplete(
+        OperationContext opContext,
+        String entityName,
+        String query,
+        String field,
+        Filter requestParams,
+        int limit) {
+      return null;
+    }
+
+    @Override
+    public Map<String, Long> aggregateByValue(
+        OperationContext opContext,
+        List<String> entityNames,
+        String field,
+        Filter requestParams,
+        int limit) {
+      return null;
+    }
+
+    @Override
+    public BrowseResult browse(
+        OperationContext opContext,
+        String entityName,
+        String path,
+        Filter requestParams,
+        int from,
+        int size) {
+      return null;
+    }
+
+    @Override
+    public BrowseResultV2 browseV2(
+        OperationContext opContext,
+        String entityName,
+        String path,
+        Filter filter,
+        String input,
+        int start,
+        int count) {
+      return null;
+    }
+
+    @Nonnull
+    @Override
+    public BrowseResultV2 browseV2(
+        @Nonnull OperationContext opContext,
+        @Nonnull List<String> entityNames,
+        @Nonnull String path,
+        @Nullable Filter filter,
+        @Nonnull String input,
+        int start,
+        int count) {
+      return null;
+    }
+
+    @Override
+    public List<String> getBrowsePaths(OperationContext opContext, String entityName, Urn urn) {
+      return null;
+    }
+
+    @Override
+    public ScrollResult fullTextScroll(
+        OperationContext opContext,
+        List<String> entities,
+        String input,
+        Filter postFilters,
+        List<SortCriterion> sortCriteria,
+        String scrollId,
+        String keepAlive,
+        int size) {
+      return null;
+    }
+
+    @Override
+    public ScrollResult structuredScroll(
+        OperationContext opContext,
+        List<String> entities,
+        String input,
+        Filter postFilters,
+        List<SortCriterion> sortCriteria,
+        String scrollId,
+        String keepAlive,
+        int size) {
+      return null;
+    }
+
+    @Override
+    public int maxResultSize() {
+      return 0;
+    }
+
+    @Override
+    public ExplainResponse explain(
+        OperationContext opContext,
+        String query,
+        String documentId,
+        String entityName,
+        Filter postFilters,
+        List<SortCriterion> sortCriteria,
+        String scrollId,
+        String keepAlive,
+        int size,
+        List<String> facets) {
+      return null;
+    }
+
+    @Override
+    public IndexConvention getIndexConvention() {
+      return null;
+    }
+  }
+}
diff --git a/metadata-service/servlet/build.gradle b/metadata-service/servlet/build.gradle
index f961bf6a9de7e..725352e69d52b 100644
--- a/metadata-service/servlet/build.gradle
+++ b/metadata-service/servlet/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'java'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(':metadata-io')
   implementation project(':datahub-graphql-core')
diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle
index 80265e8c632de..5e00207c1726e 100644
--- a/metadata-service/war/build.gradle
+++ b/metadata-service/war/build.gradle
@@ -3,6 +3,7 @@ plugins {
   id 'com.palantir.docker'
 }
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
 apply from: "../../gradle/versioning/versioning.gradle"
 
 ext {
@@ -43,6 +44,10 @@ dependencies {
   implementation externalDependency.awsMskIamAuth
   testRuntimeOnly externalDependency.logbackClassic
   implementation externalDependency.charle
+
+  testImplementation externalDependency.testng
+  testImplementation externalDependency.springBootTest
+  testRuntimeOnly externalDependency.logbackClassic
 }
 configurations.all{
   exclude group: "com.charleskorn.kaml", module:"kaml"
diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/CommonApplicationConfig.java b/metadata-service/war/src/main/java/com/linkedin/gms/CommonApplicationConfig.java
index e47a2b4e278e4..69fb9df2b04db 100644
--- a/metadata-service/war/src/main/java/com/linkedin/gms/CommonApplicationConfig.java
+++ b/metadata-service/war/src/main/java/com/linkedin/gms/CommonApplicationConfig.java
@@ -40,7 +40,8 @@
       "com.linkedin.gms.factory.plugins",
       "com.linkedin.gms.factory.change",
       "com.datahub.event.hook",
-      "com.linkedin.gms.factory.notifications"
+      "com.linkedin.gms.factory.notifications",
+      "com.linkedin.gms.factory.telemetry"
     })
 @PropertySource(value = "classpath:/application.yaml", factory = YamlPropertySourceFactory.class)
 @Configuration
diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json
index e0f26b908c499..e01b4f6f47eaa 100644
--- a/metadata-service/war/src/main/resources/boot/policies.json
+++ b/metadata-service/war/src/main/resources/boot/policies.json
@@ -36,6 +36,7 @@
         "CREATE_BUSINESS_ATTRIBUTE",
         "MANAGE_BUSINESS_ATTRIBUTE",
         "MANAGE_STRUCTURED_PROPERTIES",
+        "VIEW_STRUCTURED_PROPERTIES_PAGE",
         "MANAGE_DOCUMENTATION_FORMS",
         "MANAGE_FEATURES",
         "MANAGE_SYSTEM_OPERATIONS"
@@ -185,6 +186,7 @@
         "CREATE_BUSINESS_ATTRIBUTE",
         "MANAGE_BUSINESS_ATTRIBUTE",
         "MANAGE_STRUCTURED_PROPERTIES",
+        "VIEW_STRUCTURED_PROPERTIES_PAGE",
         "MANAGE_DOCUMENTATION_FORMS",
         "MANAGE_FEATURES"
       ],
@@ -274,6 +276,7 @@
         "MANAGE_TAGS",
         "MANAGE_BUSINESS_ATTRIBUTE",
         "MANAGE_STRUCTURED_PROPERTIES",
+        "VIEW_STRUCTURED_PROPERTIES_PAGE",
         "MANAGE_DOCUMENTATION_FORMS",
         "MANAGE_FEATURES"
       ],
@@ -427,7 +430,8 @@
         "GET_TIMESERIES_ASPECT_PRIVILEGE",
         "GET_ENTITY_PRIVILEGE",
         "GET_TIMELINE_PRIVILEGE",
-        "ES_EXPLAIN_QUERY_PRIVILEGE"
+        "ES_EXPLAIN_QUERY_PRIVILEGE",
+        "VIEW_STRUCTURED_PROPERTIES_PAGE"
       ],
       "displayName": "Readers - Metadata Policy",
       "description": "Readers can view all assets.",
diff --git a/metadata-service/war/src/test/java/com/linkedin/gms/SpringTest.java b/metadata-service/war/src/test/java/com/linkedin/gms/SpringTest.java
new file mode 100644
index 0000000000000..27dabf1e33ddd
--- /dev/null
+++ b/metadata-service/war/src/test/java/com/linkedin/gms/SpringTest.java
@@ -0,0 +1,32 @@
+package com.linkedin.gms;
+
+import static org.testng.AssertJUnit.assertNotNull;
+
+import com.linkedin.gms.factory.telemetry.DailyReport;
+import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import io.ebean.Database;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
+import org.testng.annotations.Test;
+
+@SpringBootTest(
+    webEnvironment = SpringBootTest.WebEnvironment.MOCK,
+    properties = {"telemetry.enabledServer=true"})
+@ContextConfiguration(classes = CommonApplicationConfig.class)
+public class SpringTest extends AbstractTestNGSpringContextTests {
+
+  // Mock Beans take precedence, we add these to avoid needing to configure data sources etc. while
+  // still testing prod config
+  @MockBean private Database database;
+  @MockBean private ConfigEntityRegistry configEntityRegistry;
+  @MockBean private EntityRegistry entityRegistry;
+
+  @Test
+  public void testTelemetry() {
+    DailyReport dailyReport = this.applicationContext.getBean(DailyReport.class);
+    assertNotNull(dailyReport);
+  }
+}
diff --git a/metadata-utils/build.gradle b/metadata-utils/build.gradle
index 919d93c5f9fe1..07ce50993655d 100644
--- a/metadata-utils/build.gradle
+++ b/metadata-utils/build.gradle
@@ -3,6 +3,8 @@ plugins {
   id 'pegasus'
 }
 
+apply from: '../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   api externalDependency.avro
   implementation externalDependency.commonsLang
diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java
index bc676e94ecd4f..80a11ab98bbf4 100644
--- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java
+++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java
@@ -169,6 +169,12 @@ public class PoliciesConfig {
           "Manage Structured Properties",
           "Manage structured properties in your instance.");
 
+  public static final Privilege VIEW_STRUCTURED_PROPERTIES_PAGE_PRIVILEGE =
+      Privilege.of(
+          "VIEW_STRUCTURED_PROPERTIES_PAGE",
+          "View Structured Properties",
+          "View structured properties in your instance.");
+
   public static final Privilege MANAGE_DOCUMENTATION_FORMS_PRIVILEGE =
       Privilege.of(
           "MANAGE_DOCUMENTATION_FORMS",
@@ -213,6 +219,7 @@ public class PoliciesConfig {
           MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE,
           MANAGE_CONNECTIONS_PRIVILEGE,
           MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE,
+          VIEW_STRUCTURED_PROPERTIES_PAGE_PRIVILEGE,
           MANAGE_DOCUMENTATION_FORMS_PRIVILEGE,
           MANAGE_FEATURES_PRIVILEGE,
           MANAGE_SYSTEM_OPERATIONS_PRIVILEGE);
diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/CriterionUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/CriterionUtils.java
index e40c4af1e0ae7..f8e138487fc16 100644
--- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/CriterionUtils.java
+++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/CriterionUtils.java
@@ -1,17 +1,81 @@
 package com.linkedin.metadata.utils;
 
+import static com.linkedin.metadata.Constants.URN_LI_PREFIX;
+
 import com.linkedin.data.template.StringArray;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.Criterion;
+import com.linkedin.metadata.query.filter.Filter;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 public class CriterionUtils {
   private CriterionUtils() {}
 
+  /**
+   * This function is meant to validate and correct Filter input for rest.li endpoints.
+   *
+   * @param inputFilter the rest.li filter parameter
+   * @return validated and corrected filter
+   */
+  @Nullable
+  public static Filter validateAndConvert(@Nullable Filter inputFilter) {
+    if (inputFilter != null) {
+      List<Criterion> invalidCriterion = new ArrayList<>();
+      if (inputFilter.hasCriteria()) {
+        invalidCriterion.addAll(
+            inputFilter.getCriteria().stream()
+                .filter(
+                    criterion ->
+                        (criterion.hasValue() && !criterion.getValue().isEmpty())
+                            || !criterion.hasValue())
+                .collect(Collectors.toList()));
+      }
+      if (inputFilter.hasOr()) {
+        invalidCriterion.addAll(
+            inputFilter.getOr().stream()
+                .flatMap(c -> c.getAnd().stream())
+                .filter(
+                    criterion ->
+                        (criterion.hasValue() && !criterion.getValue().isEmpty())
+                            || !criterion.hasValue())
+                .collect(Collectors.toList()));
+      }
+
+      for (Criterion criterion : invalidCriterion) {
+        if (criterion.hasValue()) {
+          if ((criterion.getValue().contains(",")
+                  && !criterion.getValue().startsWith(URN_LI_PREFIX))
+              || criterion.getValue().contains(")," + URN_LI_PREFIX)) {
+            throw new IllegalArgumentException(
+                "Criterion `value` is deprecated and contains an ambiguous comma. Please use `values`.");
+          }
+          if (criterion.hasValues() && !criterion.getValue().equals(criterion.getValues().get(0))) {
+            throw new IllegalArgumentException(
+                "Criterion `value` is deprecated and `values`[0] is populated with a conflicting value.");
+          }
+          // auto-convert
+          if (!criterion.hasValues()) {
+            log.error(
+                "Deprecated use of a filter using Criterion's `value` has been detected and corrected. Please migrate to `values` instead.");
+            criterion.setValues(new StringArray(criterion.getValue()));
+          }
+        }
+        // must be set per required field
+        criterion.setValue("");
+      }
+    }
+    return inputFilter;
+  }
+
   public static Criterion buildExistsCriterion(@Nonnull String field) {
     return buildCriterion(field, Condition.EXISTS, false, Collections.emptyList());
   }
diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java
index 6638481c1d279..993edc44daeff 100644
--- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java
+++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java
@@ -1,8 +1,10 @@
 package com.linkedin.metadata.utils;
 
 import com.datahub.util.RecordUtils;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.ByteString;
+import com.linkedin.data.DataMap;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.entity.Aspect;
 import com.linkedin.entity.EntityResponse;
@@ -12,6 +14,8 @@
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.mxe.GenericAspect;
 import com.linkedin.mxe.GenericPayload;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -22,6 +26,22 @@ public class GenericRecordUtils {
 
   private GenericRecordUtils() {}
 
+  public static <T extends RecordTemplate> T copy(T input, Class<T> clazz) {
+    try {
+      if (input == null) {
+        return null;
+      }
+      Constructor<T> constructor = clazz.getConstructor(DataMap.class);
+      return constructor.newInstance(input.data().copy());
+    } catch (CloneNotSupportedException
+        | InvocationTargetException
+        | NoSuchMethodException
+        | InstantiationException
+        | IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   /** Deserialize the given value into the aspect based on the input aspectSpec */
   @Nonnull
   public static RecordTemplate deserializeAspect(
@@ -70,6 +90,14 @@ public static GenericAspect serializeAspect(@Nonnull String str) {
     return genericAspect;
   }
 
+  @Nonnull
+  public static GenericAspect serializeAspect(@Nonnull JsonNode json) {
+    GenericAspect genericAspect = new GenericAspect();
+    genericAspect.setValue(ByteString.unsafeWrap(json.toString().getBytes(StandardCharsets.UTF_8)));
+    genericAspect.setContentType(GenericRecordUtils.JSON);
+    return genericAspect;
+  }
+
   @Nonnull
   public static GenericPayload serializePayload(@Nonnull RecordTemplate payload) {
     GenericPayload genericPayload = new GenericPayload();
diff --git a/metadata-utils/src/test/java/com/linkedin/metadata/utils/CriterionUtilsTest.java b/metadata-utils/src/test/java/com/linkedin/metadata/utils/CriterionUtilsTest.java
new file mode 100644
index 0000000000000..e2f22dd665c7c
--- /dev/null
+++ b/metadata-utils/src/test/java/com/linkedin/metadata/utils/CriterionUtilsTest.java
@@ -0,0 +1,274 @@
+package com.linkedin.metadata.utils;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+import com.linkedin.data.template.StringArray;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.Criterion;
+import com.linkedin.metadata.query.filter.CriterionArray;
+import com.linkedin.metadata.query.filter.Filter;
+import org.testng.annotations.Test;
+
+public class CriterionUtilsTest {
+  @Test
+  public void testNullFilter() {
+    Filter result = CriterionUtils.validateAndConvert(null);
+    assertNull(result);
+  }
+
+  @Test
+  public void testEmptyFilter() {
+    Filter input = new Filter();
+    Filter result = CriterionUtils.validateAndConvert(input);
+    assertNotNull(result);
+    assertFalse(result.hasCriteria());
+    assertFalse(result.hasOr());
+  }
+
+  @Test
+  public void testSimpleCriterionConversion() {
+    Filter input = new Filter();
+    Criterion criterion = new Criterion();
+    criterion.setValue("testValue");
+    input.setCriteria(new CriterionArray(criterion));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    Criterion convertedCriterion = result.getCriteria().get(0);
+    assertEquals(convertedCriterion.getValue(), "");
+    assertTrue(convertedCriterion.hasValues());
+    assertEquals("testValue", convertedCriterion.getValues().get(0));
+  }
+
+  @Test
+  public void testOrClauseCriterionConversion() {
+    Filter input = new Filter();
+
+    // Create OR clause with AND criteria
+    Criterion criterion = new Criterion();
+    criterion.setValue("orValue");
+
+    ConjunctiveCriterion conjunctive = new ConjunctiveCriterion();
+    conjunctive.setAnd(new CriterionArray(criterion));
+
+    input.setOr(new ConjunctiveCriterionArray(conjunctive));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    Criterion convertedCriterion = result.getOr().get(0).getAnd().get(0);
+    assertEquals(convertedCriterion.getValue(), "");
+    assertTrue(convertedCriterion.hasValues());
+    assertEquals("orValue", convertedCriterion.getValues().get(0));
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testCommaInValueThrowsException() {
+    Filter input = new Filter();
+    Criterion criterion = new Criterion();
+    criterion.setValue("value1,value2");
+    input.setCriteria(new CriterionArray(criterion));
+
+    CriterionUtils.validateAndConvert(input);
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testConflictingValuesThrowsException() {
+    Filter input = new Filter();
+    Criterion criterion = new Criterion();
+    criterion.setValue("value1");
+    criterion.setValues(new StringArray("differentValue"));
+    input.setCriteria(new CriterionArray(criterion));
+
+    CriterionUtils.validateAndConvert(input);
+  }
+
+  @Test
+  public void testExistingValuesNotModified() {
+    Filter input = new Filter();
+    Criterion criterion = new Criterion();
+    criterion.setValue("value1");
+    criterion.setValues(new StringArray("value1")); // Same value, should not throw exception
+    input.setCriteria(new CriterionArray(criterion));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    Criterion convertedCriterion = result.getCriteria().get(0);
+    assertEquals(convertedCriterion.getValue(), "");
+    assertTrue(convertedCriterion.hasValues());
+    assertEquals("value1", convertedCriterion.getValues().get(0));
+  }
+
+  @Test
+  public void testMultipleCriteriaConversion() {
+    Filter input = new Filter();
+
+    Criterion criterion1 = new Criterion();
+    criterion1.setValue("value1");
+
+    Criterion criterion2 = new Criterion();
+    criterion2.setValue("value2");
+
+    input.setCriteria(new CriterionArray(criterion1, criterion2));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    assertEquals(2, result.getCriteria().size());
+
+    for (Criterion c : result.getCriteria()) {
+      assertEquals(c.getValue(), "");
+      assertTrue(c.hasValues());
+      assertTrue(c.getValues().get(0).equals("value1") || c.getValues().get(0).equals("value2"));
+    }
+  }
+
+  @Test
+  public void testMixedCriteriaAndOrClause() {
+    Filter input = new Filter();
+
+    // Add direct criteria
+    Criterion criterion1 = new Criterion();
+    criterion1.setValue("directValue");
+    input.setCriteria(new CriterionArray(criterion1));
+
+    // Add OR clause with AND criteria
+    Criterion criterion2 = new Criterion();
+    criterion2.setValue("orValue");
+    ConjunctiveCriterion conjunctive = new ConjunctiveCriterion();
+    conjunctive.setAnd(new CriterionArray(criterion2));
+    input.setOr(new ConjunctiveCriterionArray(conjunctive));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    // Check direct criterion
+    Criterion convertedDirect = result.getCriteria().get(0);
+    assertEquals(convertedDirect.getValue(), "");
+    assertTrue(convertedDirect.hasValues());
+    assertEquals("directValue", convertedDirect.getValues().get(0));
+
+    // Check OR clause criterion
+    Criterion convertedOr = result.getOr().get(0).getAnd().get(0);
+    assertEquals(convertedOr.getValue(), "");
+    assertTrue(convertedOr.hasValues());
+    assertEquals("orValue", convertedOr.getValues().get(0));
+  }
+
+  @Test
+  public void testEmptyStringValueNotConverted() {
+    Filter input = new Filter();
+    Criterion criterion = new Criterion();
+    criterion.setValue(""); // Empty string value
+    input.setCriteria(new CriterionArray(criterion));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    Criterion convertedCriterion = result.getCriteria().get(0);
+    assertEquals(convertedCriterion.getValue(), "");
+    assertFalse(convertedCriterion.hasValues()); // Should not be converted since value was empty
+  }
+
+  @Test
+  public void testMixedEmptyAndNonEmptyValues() {
+    Filter input = new Filter();
+
+    Criterion emptyCriterion = new Criterion();
+    emptyCriterion.setValue("");
+
+    Criterion nonEmptyCriterion = new Criterion();
+    nonEmptyCriterion.setValue("value1");
+
+    input.setCriteria(new CriterionArray(emptyCriterion, nonEmptyCriterion));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    assertEquals(2, result.getCriteria().size());
+
+    // Check empty criterion
+    Criterion convertedEmpty = result.getCriteria().get(0);
+    assertEquals(convertedEmpty.getValue(), "");
+    assertFalse(convertedEmpty.hasValues());
+
+    // Check non-empty criterion
+    Criterion convertedNonEmpty = result.getCriteria().get(1);
+    assertEquals(convertedNonEmpty.getValue(), "");
+    assertTrue(convertedNonEmpty.hasValues());
+    assertEquals(convertedNonEmpty.getValues().get(0), "value1");
+  }
+
+  @Test
+  public void testOrClauseWithEmptyValues() {
+    Filter input = new Filter();
+
+    // Create OR clause with mixed empty and non-empty criteria
+    Criterion emptyCriterion = new Criterion();
+    emptyCriterion.setValue("");
+
+    Criterion nonEmptyCriterion = new Criterion();
+    nonEmptyCriterion.setValue("orValue");
+
+    ConjunctiveCriterion conjunctive = new ConjunctiveCriterion();
+    conjunctive.setAnd(new CriterionArray(emptyCriterion, nonEmptyCriterion));
+
+    input.setOr(new ConjunctiveCriterionArray(conjunctive));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    // Check empty criterion
+    Criterion convertedEmpty = result.getOr().get(0).getAnd().get(0);
+    assertEquals(convertedEmpty.getValue(), "");
+    assertFalse(convertedEmpty.hasValues());
+
+    // Check non-empty criterion
+    Criterion convertedNonEmpty = result.getOr().get(0).getAnd().get(1);
+    assertEquals(convertedNonEmpty.getValue(), "");
+    assertTrue(convertedNonEmpty.hasValues());
+    assertEquals(convertedNonEmpty.getValues().get(0), "orValue");
+  }
+
+  @Test
+  public void testCriterionWithOnlyValues() {
+    Filter input = new Filter();
+    Criterion criterion = new Criterion();
+    criterion.setValues(new StringArray("value1")); // Only has values, no value field set
+    input.setCriteria(new CriterionArray(criterion));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    Criterion convertedCriterion = result.getCriteria().get(0);
+    assertEquals(convertedCriterion.getValue(), "");
+    assertTrue(convertedCriterion.hasValues());
+    assertEquals(convertedCriterion.getValues().get(0), "value1");
+  }
+
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testMultiUrnThrowsException() {
+    Filter input = new Filter();
+    Criterion criterion = new Criterion();
+    criterion.setValue(
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,foo,PROD),urn:li:dataset:(urn:li:dataPlatform:postgres,foo,PROD)");
+    input.setCriteria(new CriterionArray(criterion));
+
+    CriterionUtils.validateAndConvert(input);
+  }
+
+  @Test
+  public void testUrnConversion() {
+    Filter input = new Filter();
+    Criterion criterion = new Criterion();
+    criterion.setValue("urn:li:dataset:(urn:li:dataPlatform:postgres,foo,PROD)");
+    input.setCriteria(new CriterionArray(criterion));
+
+    Filter result = CriterionUtils.validateAndConvert(input);
+
+    Criterion convertedCriterion = result.getCriteria().get(0);
+    assertEquals(convertedCriterion.getValue(), "");
+    assertTrue(convertedCriterion.hasValues());
+    assertEquals(
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,foo,PROD)",
+        convertedCriterion.getValues().get(0));
+  }
+}
diff --git a/mock-entity-registry/build.gradle b/mock-entity-registry/build.gradle
index 8242d6451dd60..3a6ce0eedcba6 100644
--- a/mock-entity-registry/build.gradle
+++ b/mock-entity-registry/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'java'
 }
 
+apply from: '../gradle/coverage/java-coverage.gradle'
+
 dependencies {
     implementation project(':entity-registry')
 }
\ No newline at end of file
diff --git a/python-build/.gitignore b/python-build/.gitignore
new file mode 100644
index 0000000000000..d2de6dec25809
--- /dev/null
+++ b/python-build/.gitignore
@@ -0,0 +1,3 @@
+
+/wheels
+/site
diff --git a/python-build/build.gradle b/python-build/build.gradle
new file mode 100644
index 0000000000000..e90bffd46828c
--- /dev/null
+++ b/python-build/build.gradle
@@ -0,0 +1,27 @@
+plugins {
+  id 'base'
+}
+
+ext {
+  python_executable = 'python3'
+}
+
+task checkPythonVersion(type: Exec) {
+  commandLine python_executable, '-c',
+    'import sys; sys.version_info >= (3, 8), f"Python version {sys.version_info} is too old"'
+}
+
+task buildWheels(type: Exec, dependsOn: [
+  checkPythonVersion,
+  ':metadata-ingestion:buildWheel',
+  ':metadata-ingestion-modules:airflow-plugin:buildWheel',
+  ':metadata-ingestion-modules:dagster-plugin:buildWheel',
+  ':metadata-ingestion-modules:prefect-plugin:buildWheel',
+  ':metadata-ingestion-modules:gx-plugin:buildWheel',
+]) {
+  commandLine python_executable, "copy_wheels.py"
+}
+
+task buildSite(type: Exec, dependsOn: [buildWheels]) {
+  commandLine python_executable, "build_site.py"
+}
diff --git a/python-build/build_site.py b/python-build/build_site.py
new file mode 100644
index 0000000000000..73941eca9968c
--- /dev/null
+++ b/python-build/build_site.py
@@ -0,0 +1,150 @@
+import contextlib
+import json
+import os
+import pathlib
+import shutil
+import subprocess
+from datetime import datetime, timezone
+
+PYTHON_BUILD_DIR = pathlib.Path(__file__).parent
+WHEEL_DIR = PYTHON_BUILD_DIR / "wheels"
+SITE_OUTPUT_DIR = PYTHON_BUILD_DIR / "site"
+
+shutil.rmtree(SITE_OUTPUT_DIR, ignore_errors=True)
+SITE_OUTPUT_DIR.mkdir(parents=True)
+
+SITE_ARTIFACT_WHEEL_DIR = SITE_OUTPUT_DIR / "artifacts" / "wheels"
+SITE_ARTIFACT_WHEEL_DIR.mkdir(parents=True)
+for wheel_file in WHEEL_DIR.glob("*"):
+    shutil.copy(wheel_file, SITE_ARTIFACT_WHEEL_DIR)
+
+
+def package_name(wheel_file: pathlib.Path) -> str:
+    return wheel_file.name.split("-")[0].replace("_", "-")
+
+
+# Get some extra context about the build
+ts = datetime.now(timezone.utc).isoformat()
+context_info: dict = {
+    "timestamp": ts,
+}
+
+# Get branch info.
+with contextlib.suppress(Exception):
+    if branch_info := os.getenv("GITHUB_HEAD_REF"):
+        pass
+    else:
+        branch_info = subprocess.check_output(
+            ["git", "branch", "--show-current"], text=True
+        )
+    context_info["branch"] = branch_info.strip()
+
+# Get commit info.
+with contextlib.suppress(Exception):
+    commit_info = subprocess.check_output(
+        ["git", "log", "-1", "--pretty=%H%n%B"], text=True
+    )
+    commit_hash, commit_msg = commit_info.strip().split("\n", 1)
+    context_info["commit"] = {
+        "hash": commit_hash,
+        "message": commit_msg.strip(),
+    }
+
+# Get PR info.
+with contextlib.suppress(Exception):
+    pr_info = "unknown"
+    if github_ref := os.getenv("GITHUB_REF"):
+        # e.g. GITHUB_REF=refs/pull/12157/merge
+        parts = github_ref.split("/")
+        if parts[1] == "pull":
+            pull_number = parts[2]
+            pr_info = json.loads(
+                subprocess.check_output(
+                    ["gh", "pr", "view", pull_number, "--json", "title,number,url"],
+                    text=True,
+                )
+            )
+    else:
+        # The `gh` CLI might be able to figure it out.
+        pr_info = json.loads(
+            subprocess.check_output(
+                ["gh", "pr", "view", "--json", "title,number,url"], text=True
+            )
+        )
+    context_info["pr"] = pr_info
+
+
+newline = "\n"
+(SITE_OUTPUT_DIR / "index.html").write_text(
+    f"""
+<html>
+  <head>
+    <title>DataHub Python Builds</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
+
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/semantic-ui@2.5.0/dist/semantic.min.css" integrity="sha256-cDGQ39yChhpN5vzgHbjIdGEtQ5kXE9tttCsI7VR9TuY=" crossorigin="anonymous">
+    <script src="https://cdn.jsdelivr.net/npm/semantic-ui@2.5.0/dist/semantic.min.js" integrity="sha256-fN8vcX2ULyTDspVTHEteK8hd3rQAb5thNiwakjAW75Q=" crossorigin="anonymous"></script>
+
+    <!-- CDN example (jsDelivr) -->
+    <script src="https://cdn.jsdelivr.net/npm/dayjs@1.11.13/dayjs.min.js" integrity="sha256-nP25Pzivzy0Har7NZtMr/TODzfGWdlTrwmomYF2vQXM=" crossorigin="anonymous"></script>
+    <script src="https://cdn.jsdelivr.net/npm/dayjs@1.11.13/plugin/relativeTime.js" integrity="sha256-muryXOPFkVJcJO1YFmhuKyXYmGDT2TYVxivG0MCgRzg=" crossorigin="anonymous"></script>
+    <script>dayjs.extend(window.dayjs_plugin_relativeTime)</script>
+  </head>
+  <body>
+    <div class="ui container">
+      <h1 class="ui header" style="padding-top: 1.5em;">DataHub Python Builds</h1>
+      <p>
+        These prebuilt wheel files can be used to install our Python packages as of a specific commit.
+      </p>
+
+      <h2>Build context</h2>
+      <p>
+        Built <span id="build-timestamp">at {ts}</span>.
+      </p>
+      <pre id="context-info">{json.dumps(context_info, indent=2)}</pre>
+
+      <h2>Usage</h2>
+      <p>
+      Current base URL: <span class="base-url">unknown</span>
+      </p>
+
+      <table class="ui celled table">
+      <thead>
+        <tr>
+          <th>Package</th>
+          <th>Size</th>
+          <th>Install command</th>
+        </tr>
+      </thead>
+      <tbody>
+        {
+        newline.join(
+            f'''
+            <tr>
+              <td><code>{package_name(wheel_file)}</code></td>
+              <td>{wheel_file.stat().st_size / 1024 / 1024:.3f} MB</td>
+              <td><code>uv pip install '{package_name(wheel_file)} @ <span class="base-url">&lt;base-url&gt;</span>/artifacts/wheels/{wheel_file.name}'</code></td>
+            </tr>
+            '''
+            for wheel_file in sorted(WHEEL_DIR.glob("*.whl"))
+        )
+        }
+      </tbody>
+      </table>
+    </div>
+  </body>
+  <script>
+    const baseUrl = window.location.href.split('/').slice(0, -1).join('/');
+    document.querySelectorAll(".base-url").forEach(el => {{
+      el.textContent = baseUrl;
+    }});
+
+    const buildTimestamp = document.getElementById("build-timestamp");
+    const buildTimestampDate = dayjs('{ts}');
+    buildTimestamp.textContent = buildTimestampDate.fromNow();
+  </script>
+</html>
+"""
+)
+
+print("DataHub Python wheel site built in", SITE_OUTPUT_DIR)
diff --git a/python-build/copy_wheels.py b/python-build/copy_wheels.py
new file mode 100644
index 0000000000000..b66662cbfe991
--- /dev/null
+++ b/python-build/copy_wheels.py
@@ -0,0 +1,27 @@
+import pathlib
+import shutil
+
+PYTHON_BUILD_DIR = pathlib.Path(__file__).parent
+ROOT_DIR = PYTHON_BUILD_DIR.parent
+WHEEL_OUTPUT_DIR = PYTHON_BUILD_DIR / "wheels"
+
+# These should line up with the build.gradle file.
+wheel_dirs = [
+    ROOT_DIR / "metadata-ingestion/dist",
+    ROOT_DIR / "metadata-ingestion-modules/airflow-plugin/dist",
+    ROOT_DIR / "metadata-ingestion-modules/dagster-plugin/dist",
+    ROOT_DIR / "metadata-ingestion-modules/prefect-plugin/dist",
+    ROOT_DIR / "metadata-ingestion-modules/gx-plugin/dist",
+]
+
+# Delete and recreate the output directory.
+if WHEEL_OUTPUT_DIR.exists():
+    shutil.rmtree(WHEEL_OUTPUT_DIR)
+WHEEL_OUTPUT_DIR.mkdir(parents=True)
+
+# Copy things over.
+for wheel_dir in wheel_dirs:
+    for wheel_file in wheel_dir.glob("*"):
+        shutil.copy(wheel_file, WHEEL_OUTPUT_DIR)
+
+print("Copied wheels to", WHEEL_OUTPUT_DIR)
diff --git a/settings.gradle b/settings.gradle
index fa1fdb9f1a67c..b0c2c707d566c 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -64,6 +64,7 @@ include 'metadata-ingestion-modules:airflow-plugin'
 include 'metadata-ingestion-modules:gx-plugin'
 include 'metadata-ingestion-modules:dagster-plugin'
 include 'metadata-ingestion-modules:prefect-plugin'
+include 'python-build'
 include 'smoke-test'
 include 'metadata-auth:auth-api'
 include 'metadata-service:schema-registry-api'
@@ -75,3 +76,5 @@ include 'metadata-service:configuration'
 include ':metadata-jobs:common'
 include ':metadata-operation-context'
 include ':metadata-service:openapi-servlet:models'
+include ':metadata-integration:java:datahub-schematron:lib'
+include ':metadata-integration:java:datahub-schematron:cli'
diff --git a/smoke-test/build.gradle b/smoke-test/build.gradle
index def3e814b2ba0..73ecdcb08ea14 100644
--- a/smoke-test/build.gradle
+++ b/smoke-test/build.gradle
@@ -16,7 +16,7 @@ node {
   }
 
   // Version of node to use.
-  version = '21.2.0'
+  version = '22.12.0'
 
   // Version of Yarn to use.
   yarnVersion = '1.22.22'
diff --git a/smoke-test/conftest.py b/smoke-test/conftest.py
index 69a58cd766182..6d148db9886a4 100644
--- a/smoke-test/conftest.py
+++ b/smoke-test/conftest.py
@@ -15,16 +15,19 @@
 os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false"
 
 
+def build_auth_session():
+    wait_for_healthcheck_util(requests)
+    return TestSessionWrapper(get_frontend_session())
+
+
 @pytest.fixture(scope="session")
 def auth_session():
-    wait_for_healthcheck_util(requests)
-    auth_session = TestSessionWrapper(get_frontend_session())
+    auth_session = build_auth_session()
     yield auth_session
     auth_session.destroy()
 
 
-@pytest.fixture(scope="session")
-def graph_client(auth_session) -> DataHubGraph:
+def build_graph_client(auth_session):
     print(auth_session.cookies)
     graph: DataHubGraph = DataHubGraph(
         config=DatahubClientConfig(
@@ -34,6 +37,11 @@ def graph_client(auth_session) -> DataHubGraph:
     return graph
 
 
+@pytest.fixture(scope="session")
+def graph_client(auth_session) -> DataHubGraph:
+    return build_graph_client(auth_session)
+
+
 def pytest_sessionfinish(session, exitstatus):
     """whole test run finishes."""
     send_message(exitstatus)
diff --git a/smoke-test/cypress-dev.sh b/smoke-test/cypress-dev.sh
index 3db81b11c67fa..bb9b287cb654e 100755
--- a/smoke-test/cypress-dev.sh
+++ b/smoke-test/cypress-dev.sh
@@ -16,7 +16,16 @@ set -x
 # set environment variables for the test
 source ./set-test-env-vars.sh
 
-python -c 'from tests.cypress.integration_test import ingest_data; ingest_data()'
+LOAD_DATA=$(cat <<EOF
+from conftest import build_auth_session, build_graph_client
+from tests.cypress.integration_test import ingest_data
+
+auth_session = build_auth_session()
+ingest_data(auth_session, build_graph_client(auth_session))
+EOF
+)
+
+echo -e "$LOAD_DATA" | python
 
 cd tests/cypress
 yarn install
diff --git a/smoke-test/pytest.ini b/smoke-test/pytest.ini
index 61ce840fd713c..3762344cec4be 100644
--- a/smoke-test/pytest.ini
+++ b/smoke-test/pytest.ini
@@ -1,3 +1,6 @@
 [pytest]
 markers =
     read_only: marks tests as read only (deselect with '-m "not read_only"')
+    ; no_cypress_suite0: main smoke tests; expressed as the negative of the others
+    no_cypress_suite1: main smoke tests, suite 1
+    test_run_cypress: run cypress tests
diff --git a/smoke-test/test_e2e.py b/smoke-test/test_e2e.py
index 51559a6bcdff0..a1381608060b9 100644
--- a/smoke-test/test_e2e.py
+++ b/smoke-test/test_e2e.py
@@ -26,9 +26,7 @@
 )
 
 bootstrap_sample_data = "../metadata-ingestion/examples/mce_files/bootstrap_mce.json"
-usage_sample_data = (
-    "./test_resources/bigquery_usages_golden.json"
-)
+usage_sample_data = "./test_resources/bigquery_usages_golden.json"
 bq_sample_data = "./sample_bq_data.json"
 restli_default_headers = {
     "X-RestLi-Protocol-Version": "2.0.0",
diff --git a/smoke-test/tests/consistency_utils.py b/smoke-test/tests/consistency_utils.py
index 1eddc46bb220b..9179432a3f16e 100644
--- a/smoke-test/tests/consistency_utils.py
+++ b/smoke-test/tests/consistency_utils.py
@@ -3,10 +3,9 @@
 import subprocess
 import time
 
-_ELASTIC_BUFFER_WRITES_TIME_IN_SEC: int = 1
 USE_STATIC_SLEEP: bool = bool(os.getenv("USE_STATIC_SLEEP", False))
 ELASTICSEARCH_REFRESH_INTERVAL_SECONDS: int = int(
-    os.getenv("ELASTICSEARCH_REFRESH_INTERVAL_SECONDS", 5)
+    os.getenv("ELASTICSEARCH_REFRESH_INTERVAL_SECONDS", 1)
 )
 KAFKA_BOOTSTRAP_SERVER: str = str(os.getenv("KAFKA_BOOTSTRAP_SERVER", "broker:29092"))
 
@@ -71,4 +70,4 @@ def wait_for_writes_to_sync(max_timeout_in_sec: int = 120) -> None:
         )
     else:
         # we want to sleep for an additional period of time for Elastic writes buffer to clear
-        time.sleep(_ELASTIC_BUFFER_WRITES_TIME_IN_SEC)
+        time.sleep(ELASTICSEARCH_REFRESH_INTERVAL_SECONDS)
diff --git a/smoke-test/tests/cypress/README.txt b/smoke-test/tests/cypress/README.txt
index 1210e08b861e0..edd40203fe55e 100644
--- a/smoke-test/tests/cypress/README.txt
+++ b/smoke-test/tests/cypress/README.txt
@@ -1,18 +1,7 @@
 # Quick Run Tests with UI
 
-cd smoke-test/
-./cypress-dev.sh
-
-# Running Cypress Tests Locally
-
-1. Make sure the packages are installed. It uses some node modules to run locally. Run `yarn install` from this directory. If you don't have `yarn`, download it.
-
-2. Cypress tests run against your local deployment of datahub. They are dependent on the data inside. There is sample cypress data in this directory. Ideally you want to delete all the data in your local instance before ingesting the cypress data as it will throw off a few tests (like the search tests) if you have data cypress is not expecting. However, most tests will still pass- most visit specific entity pages.
-
-3. Ingest the cypress data! Using datahub cli, run `datahub ingest -c example_to_datahub_rest.yml` and then `datahub ingest -c example_siblings_to_datahub_rest.yml`.
-
-4. Set the port that you want to run your cypress tests against in ./cypress-config.json. The default is 9002- if you are developing on react locally, you probably want 3000. Do not commit this change to github.
-
-5. Now, start the local cypress server: `npx cypress open`.
-
+1. Run quickstart
+   $ ./gradlew quickstartDebug
 
+2. Load Cypress fixture data and run Cypress UI
+   $ ./gradlew cypressDev
diff --git a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js
index fb772bd7af1e7..57617d7721e59 100644
--- a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js
+++ b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js
@@ -98,7 +98,7 @@ describe("siblings", () => {
 
   it("will combine results in search", () => {
     cy.login();
-    cy.visit("/search?page=1&query=raw_orders");
+    cy.visit("/search?page=1&query=%22raw_orders%22");
 
     cy.contains("Showing 1 - 2 of ");
 
diff --git a/smoke-test/tests/cypress/yarn.lock b/smoke-test/tests/cypress/yarn.lock
index 2433e9f8fae08..c6116609b1146 100644
--- a/smoke-test/tests/cypress/yarn.lock
+++ b/smoke-test/tests/cypress/yarn.lock
@@ -510,9 +510,9 @@ core-util-is@1.0.2:
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
 cross-spawn@^7.0.0, cross-spawn@^7.0.2:
-  version "7.0.3"
-  resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
+  integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
   dependencies:
     path-key "^3.1.0"
     shebang-command "^2.0.0"
diff --git a/smoke-test/tests/database/__init__.py b/smoke-test/tests/database/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/smoke-test/tests/database/test_database.py b/smoke-test/tests/database/test_database.py
new file mode 100644
index 0000000000000..656c9e98887bf
--- /dev/null
+++ b/smoke-test/tests/database/test_database.py
@@ -0,0 +1,32 @@
+import logging
+
+import pytest
+from datahub.emitter.mce_builder import make_dataset_urn
+
+from tests.utilities.concurrent_openapi import run_tests
+from tests.utils import delete_urns, wait_for_writes_to_sync
+
+logger = logging.getLogger(__name__)
+
+
+generated_urns = [make_dataset_urn("test", f"database_test_{i}") for i in range(0, 100)]
+
+
+@pytest.fixture(scope="module")
+def ingest_cleanup_data(graph_client, request):
+    print("removing test data before")
+    delete_urns(graph_client, generated_urns)
+    wait_for_writes_to_sync()
+    yield
+    print("removing test data after")
+    delete_urns(graph_client, generated_urns)
+    wait_for_writes_to_sync()
+
+
+def test_mysql_deadlock_gap_locking(auth_session, ingest_cleanup_data):
+    # This generates concurrent batches with interleaved urn ids
+    run_tests(
+        auth_session,
+        fixture_globs=["tests/database/v3/mysql_gap_deadlock/*.json"],
+        num_workers=8,
+    )
diff --git a/smoke-test/tests/database/v3/__init__.py b/smoke-test/tests/database/v3/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/__init__.py b/smoke-test/tests/database/v3/mysql_gap_deadlock/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/batchA1.json b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchA1.json
new file mode 100644
index 0000000000000..ef601dacc211c
--- /dev/null
+++ b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchA1.json
@@ -0,0 +1,115 @@
+[{
+  "request": {
+    "url": "/openapi/v3/entity/dataset",
+    "params": {
+      "async": "false"
+    },
+    "description": "Create dataset batch, single transaction",
+    "json": [
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_0,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_4,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_8,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_12,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_16,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_20,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_24,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_28,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_32,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_36,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_40,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_44,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_48,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      }
+    ]
+  }
+}]
\ No newline at end of file
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/batchA2.json b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchA2.json
new file mode 100644
index 0000000000000..3f56f730e30f5
--- /dev/null
+++ b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchA2.json
@@ -0,0 +1,107 @@
+[{
+  "request": {
+    "url": "/openapi/v3/entity/dataset",
+    "params": {
+      "async": "false"
+    },
+    "description": "Create dataset batch, single transaction",
+    "json": [
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_52,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_56,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_60,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_64,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_68,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_72,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_76,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_80,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_84,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_88,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_92,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_96,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      }
+    ]
+  }
+}]
\ No newline at end of file
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/batchB1.json b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchB1.json
new file mode 100644
index 0000000000000..de807321e815b
--- /dev/null
+++ b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchB1.json
@@ -0,0 +1,115 @@
+[{
+  "request": {
+    "url": "/openapi/v3/entity/dataset",
+    "params": {
+      "async": "false"
+    },
+    "description": "Create dataset batch, single transaction",
+    "json": [
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_1,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_5,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_9,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_13,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_17,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_21,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_25,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_29,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_33,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_37,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_41,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_45,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_49,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      }
+    ]
+  }
+}]
\ No newline at end of file
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/batchB2.json b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchB2.json
new file mode 100644
index 0000000000000..7f35e6ac24e5e
--- /dev/null
+++ b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchB2.json
@@ -0,0 +1,107 @@
+[{
+  "request": {
+    "url": "/openapi/v3/entity/dataset",
+    "params": {
+      "async": "false"
+    },
+    "description": "Create dataset batch, single transaction",
+    "json": [
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_53,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_57,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_61,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_65,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_69,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_73,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_77,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_81,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_85,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_89,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_93,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_97,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      }
+    ]
+  }
+}]
\ No newline at end of file
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/batchC1.json b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchC1.json
new file mode 100644
index 0000000000000..986c119dae29d
--- /dev/null
+++ b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchC1.json
@@ -0,0 +1,115 @@
+[{
+  "request": {
+    "url": "/openapi/v3/entity/dataset",
+    "params": {
+      "async": "false"
+    },
+    "description": "Create dataset batch, single transaction",
+    "json": [
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_2,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_6,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_10,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_14,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_18,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_22,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_26,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_30,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_34,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_38,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_42,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_46,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_50,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      }
+    ]
+  }
+}]
\ No newline at end of file
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/batchC2.json b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchC2.json
new file mode 100644
index 0000000000000..861e7cb2d0ffc
--- /dev/null
+++ b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchC2.json
@@ -0,0 +1,107 @@
+[{
+  "request": {
+    "url": "/openapi/v3/entity/dataset",
+    "params": {
+      "async": "false"
+    },
+    "description": "Create dataset batch, single transaction",
+    "json": [
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_54,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_58,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_62,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_66,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_70,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_74,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_78,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_82,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_86,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_90,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_94,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_98,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      }
+    ]
+  }
+}]
\ No newline at end of file
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/batchD1.json b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchD1.json
new file mode 100644
index 0000000000000..e4721aaeec1c3
--- /dev/null
+++ b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchD1.json
@@ -0,0 +1,115 @@
+[{
+  "request": {
+    "url": "/openapi/v3/entity/dataset",
+    "params": {
+      "async": "false"
+    },
+    "description": "Create dataset batch, single transaction",
+    "json": [
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_3,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_7,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_11,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_15,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_19,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_23,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_27,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_31,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_35,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_39,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_43,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_47,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_51,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      }
+    ]
+  }
+}]
\ No newline at end of file
diff --git a/smoke-test/tests/database/v3/mysql_gap_deadlock/batchD2.json b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchD2.json
new file mode 100644
index 0000000000000..5bb9bb3772c35
--- /dev/null
+++ b/smoke-test/tests/database/v3/mysql_gap_deadlock/batchD2.json
@@ -0,0 +1,107 @@
+[{
+  "request": {
+    "url": "/openapi/v3/entity/dataset",
+    "params": {
+      "async": "false"
+    },
+    "description": "Create dataset batch, single transaction",
+    "json": [
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_55,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_59,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_63,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_67,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_71,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_75,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_79,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_83,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_87,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_91,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_95,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      },
+      {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,database_test_99,PROD)",
+        "status": {
+          "value": {
+            "removed": false
+          }
+        }
+      }
+    ]
+  }
+}]
\ No newline at end of file
diff --git a/smoke-test/tests/lineage/test_lineage.py b/smoke-test/tests/lineage/test_lineage.py
index 8757741d1cb23..4a43fd591ae2e 100644
--- a/smoke-test/tests/lineage/test_lineage.py
+++ b/smoke-test/tests/lineage/test_lineage.py
@@ -796,7 +796,7 @@ def test_expectation(self, graph: DataHubGraph) -> bool:
                             expectation.impacted_entities[impacted_entity]
                         ), f"Expected impacted entity paths to be {expectation.impacted_entities[impacted_entity]}, found {impacted_entity_paths}"
                     except Exception:
-                        breakpoint()
+                        # breakpoint()
                         raise
                     # for i in range(len(impacted_entity_paths)):
                     #     assert impacted_entity_paths[i].path == expectation.impacted_entities[impacted_entity][i].path, f"Expected impacted entity paths to be {expectation.impacted_entities[impacted_entity][i].path}, found {impacted_entity_paths[i].path}"
diff --git a/smoke-test/tests/openapi/test_openapi.py b/smoke-test/tests/openapi/test_openapi.py
index 0217b185570be..9b753f2a06c46 100644
--- a/smoke-test/tests/openapi/test_openapi.py
+++ b/smoke-test/tests/openapi/test_openapi.py
@@ -1,91 +1,10 @@
-import concurrent.futures
-import glob
-import json
 import logging
 
-from deepdiff import DeepDiff
+from tests.utilities.concurrent_openapi import run_tests
 
 logger = logging.getLogger(__name__)
 
 
-def load_tests(fixture_glob="tests/openapi/**/*.json"):
-    for test_fixture in glob.glob(fixture_glob):
-        with open(test_fixture) as f:
-            yield (test_fixture, json.load(f))
-
-
-def execute_request(auth_session, request):
-    if "method" in request:
-        method = request.pop("method")
-    else:
-        method = "post"
-
-    url = auth_session.gms_url() + request.pop("url")
-
-    return getattr(auth_session, method)(url, **request)
-
-
-def evaluate_test(auth_session, test_name, test_data):
-    try:
-        for idx, req_resp in enumerate(test_data):
-            if "description" in req_resp["request"]:
-                description = req_resp["request"].pop("description")
-            else:
-                description = None
-            url = req_resp["request"]["url"]
-            actual_resp = execute_request(auth_session, req_resp["request"])
-            try:
-                if "response" in req_resp and "status_codes" in req_resp["response"]:
-                    assert (
-                        actual_resp.status_code in req_resp["response"]["status_codes"]
-                    )
-                else:
-                    assert actual_resp.status_code in [200, 202, 204]
-                if "response" in req_resp:
-                    if "json" in req_resp["response"]:
-                        if "exclude_regex_paths" in req_resp["response"]:
-                            exclude_regex_paths = req_resp["response"][
-                                "exclude_regex_paths"
-                            ]
-                        else:
-                            exclude_regex_paths = []
-                        diff = DeepDiff(
-                            actual_resp.json(),
-                            req_resp["response"]["json"],
-                            exclude_regex_paths=exclude_regex_paths,
-                            ignore_order=True,
-                        )
-                        assert not diff
-                    else:
-                        logger.warning("No expected response json found")
-            except Exception as e:
-                logger.error(
-                    f"Error executing step: {idx}, url: {url}, test: {test_name}"
-                )
-                if description:
-                    logger.error(f"Step {idx} Description: {description}")
-                logger.error(f"Response content: {actual_resp.content}")
-                raise e
-    except Exception as e:
-        logger.error(f"Error executing test: {test_name}")
-        raise e
-
-
-def run_tests(auth_session, fixture_globs, num_workers=3):
-    with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
-        futures = []
-        for fixture_glob in fixture_globs:
-            for test_fixture, test_data in load_tests(fixture_glob=fixture_glob):
-                futures.append(
-                    executor.submit(
-                        evaluate_test, auth_session, test_fixture, test_data
-                    )
-                )
-
-        for future in concurrent.futures.as_completed(futures):
-            logger.info(future.result())
-
-
 def test_openapi_all(auth_session):
     run_tests(auth_session, fixture_globs=["tests/openapi/*/*.json"], num_workers=10)
 
diff --git a/smoke-test/tests/openapi/v3/timeseries.json b/smoke-test/tests/openapi/v3/timeseries.json
new file mode 100644
index 0000000000000..7f4c46de25960
--- /dev/null
+++ b/smoke-test/tests/openapi/v3/timeseries.json
@@ -0,0 +1,156 @@
+[
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29",
+      "description": "Remove test dataset",
+      "method": "delete"
+    }
+  },
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29/datasetProfile",
+      "description": "Create timeseries old",
+      "params": {
+        "createIfNotExists": "false",
+        "async": "false"
+      },
+      "json": {
+        "value": {
+          "rowCount": 100,
+          "messageId": "old profile",
+          "timestampMillis": 100
+        }
+      }
+    },
+    "response": {
+      "json": {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,datasetTimeseriesV3,PROD)",
+        "datasetProfile": {
+          "value": {
+            "messageId": "old profile",
+            "timestampMillis": 100,
+            "rowCount": 100
+          }
+        }
+      }
+    }
+  },
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29/datasetProfile",
+      "description": "Create timeseries new",
+      "params": {
+        "createIfNotExists": "false",
+        "async": "false"
+      },
+      "json": {
+        "value": {
+          "rowCount": 200,
+          "messageId": "new profile",
+          "timestampMillis": 200
+        }
+      }
+    },
+    "response": {
+      "json": {
+        "urn": "urn:li:dataset:(urn:li:dataPlatform:test,datasetTimeseriesV3,PROD)",
+        "datasetProfile": {
+          "value": {
+            "messageId": "new profile",
+            "timestampMillis": 200,
+            "rowCount": 200
+          }
+        }
+      }
+    }
+  },
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29/datasetProfile",
+      "method": "get",
+      "description": "Get latest profile"
+    },
+    "response": {
+      "json": {
+        "value": {
+          "messageId": "new profile",
+          "rowCount": 200,
+          "timestampMillis": 200
+        }
+      }
+    }
+  },
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29/datasetProfile",
+      "method": "get",
+      "description": "Get older profile",
+      "params": {
+        "version": "150"
+      }
+    },
+    "response": {
+      "json": {
+        "value": {
+          "messageId": "old profile",
+          "rowCount": 100,
+          "timestampMillis": 100
+        }
+      }
+    }
+  },
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29/datasetProfile",
+      "description": "Remove test profile",
+      "method": "delete"
+    }
+  },
+  {
+    "request": {
+      "description": "Elasticsearch refresh interval",
+      "wait": 2
+    }
+  },
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29/datasetProfile",
+      "method": "get",
+      "description": "Get older profile after delete of new profile"
+    },
+    "response": {
+      "json": {
+        "value": {
+          "messageId": "old profile",
+          "rowCount": 100,
+          "timestampMillis": 100
+        }
+      }
+    }
+  },
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29/datasetProfile",
+      "description": "Remove test profile",
+      "method": "delete"
+    }
+  },
+  {
+    "request": {
+      "description": "Elasticsearch refresh interval",
+      "wait": 2
+    }
+  },
+  {
+    "request": {
+      "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2CdatasetTimeseriesV3%2CPROD%29/datasetProfile",
+      "method": "get",
+      "description": "Expected no remaining values after timeseries aspect removals"
+    },
+    "response": {
+      "status_codes": [
+        404
+      ]
+    }
+  }
+]
\ No newline at end of file
diff --git a/smoke-test/tests/patch/test_datajob_patches.py b/smoke-test/tests/patch/test_datajob_patches.py
index eb129e1e03212..e025c8a2aebeb 100644
--- a/smoke-test/tests/patch/test_datajob_patches.py
+++ b/smoke-test/tests/patch/test_datajob_patches.py
@@ -1,5 +1,7 @@
+import time
 import uuid
 
+import datahub.metadata.schema_classes as models
 from datahub.emitter.mce_builder import make_data_job_urn, make_dataset_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -136,3 +138,95 @@ def test_datajob_inputoutput_dataset_patch(graph_client):
         inputoutput_lineage_read.inputDatasetEdges[0].destinationUrn
         == other_dataset_urn
     )
+
+
+def test_datajob_multiple_inputoutput_dataset_patch(graph_client):
+    """Test creating a data job with multiple input and output datasets and verifying the aspects."""
+    # Create the data job
+    datajob_urn = "urn:li:dataJob:(urn:li:dataFlow:(airflow,training,default),training)"
+
+    # Create input and output dataset URNs
+    input_datasets = ["input_data_1", "input_data_2"]
+    output_datasets = ["output_data_1", "output_data_2"]
+
+    input_dataset_urns = [
+        make_dataset_urn(platform="s3", name=f"test_patch_{dataset}", env="PROD")
+        for dataset in input_datasets
+    ]
+    output_dataset_urns = [
+        make_dataset_urn(platform="s3", name=f"test_patch_{dataset}", env="PROD")
+        for dataset in output_datasets
+    ]
+
+    # Create edges for datasets
+    def make_edge(urn, generate_auditstamp=False):
+        audit_stamp = models.AuditStampClass(
+            time=int(time.time() * 1000.0),
+            actor="urn:li:corpuser:datahub",
+        )
+        return EdgeClass(
+            destinationUrn=str(urn),
+            lastModified=audit_stamp if generate_auditstamp else None,
+        )
+
+    # Initialize empty input/output lineage
+    initial_lineage = DataJobInputOutputClass(
+        inputDatasets=[], outputDatasets=[], inputDatasetEdges=[], outputDatasetEdges=[]
+    )
+
+    # Emit initial lineage
+    mcpw = MetadataChangeProposalWrapper(entityUrn=datajob_urn, aspect=initial_lineage)
+    graph_client.emit_mcp(mcpw)
+
+    # Create patches for input and output datasets
+    patch_builder = DataJobPatchBuilder(datajob_urn)
+    for input_urn in input_dataset_urns:
+        patch_builder.add_input_dataset(make_edge(input_urn))
+    for output_urn in output_dataset_urns:
+        patch_builder.add_output_dataset(make_edge(output_urn))
+
+    # Apply patches
+    for patch_mcp in patch_builder.build():
+        graph_client.emit_mcp(patch_mcp)
+
+    # Verify the lineage was correctly applied
+    lineage_aspect = graph_client.get_aspect(
+        entity_urn=datajob_urn,
+        aspect_type=DataJobInputOutputClass,
+    )
+
+    # Assert lineage was created
+    assert lineage_aspect is not None
+    assert lineage_aspect.inputDatasetEdges is not None
+    assert lineage_aspect.outputDatasetEdges is not None
+
+    # Verify input datasets
+    assert len(lineage_aspect.inputDatasetEdges) == len(input_datasets)
+    input_urns = {edge.destinationUrn for edge in lineage_aspect.inputDatasetEdges}
+    expected_input_urns = {str(urn) for urn in input_dataset_urns}
+    assert input_urns == expected_input_urns
+
+    # Verify output datasets
+    assert len(lineage_aspect.outputDatasetEdges) == len(output_datasets)
+    output_urns = {edge.destinationUrn for edge in lineage_aspect.outputDatasetEdges}
+    expected_output_urns = {str(urn) for urn in output_dataset_urns}
+    assert output_urns == expected_output_urns
+
+    # Test updating the same datasets again (idempotency)
+    patch_builder = DataJobPatchBuilder(datajob_urn)
+    for input_urn in input_dataset_urns:
+        patch_builder.add_input_dataset(make_edge(input_urn))
+    for output_urn in output_dataset_urns:
+        patch_builder.add_output_dataset(make_edge(output_urn))
+
+    for patch_mcp in patch_builder.build():
+        graph_client.emit_mcp(patch_mcp)
+
+    # Verify the aspect hasn't changed
+    updated_lineage_aspect = graph_client.get_aspect(
+        entity_urn=datajob_urn,
+        aspect_type=DataJobInputOutputClass,
+    )
+
+    assert updated_lineage_aspect is not None
+    assert updated_lineage_aspect.to_obj() == lineage_aspect.to_obj()
diff --git a/smoke-test/tests/platform_resources/test_platform_resource.py b/smoke-test/tests/platform_resources/test_platform_resource.py
index 09d2503179572..23a147bda695b 100644
--- a/smoke-test/tests/platform_resources/test_platform_resource.py
+++ b/smoke-test/tests/platform_resources/test_platform_resource.py
@@ -5,21 +5,17 @@
 
 import pytest
 from datahub.api.entities.platformresource.platform_resource import (
+    ElasticPlatformResourceQuery,
     PlatformResource,
     PlatformResourceKey,
+    PlatformResourceSearchFields,
 )
 
-from tests.utils import wait_for_healthcheck_util, wait_for_writes_to_sync
+from tests.utils import wait_for_writes_to_sync
 
 logger = logging.getLogger(__name__)
 
 
-@pytest.fixture(scope="module")
-def wait_for_healthchecks(auth_session):
-    wait_for_healthcheck_util(auth_session)
-    yield
-
-
 def generate_random_id(length=8):
     return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
 
@@ -42,7 +38,12 @@ def cleanup_resources(graph_client):
             logger.warning(f"Failed to delete resource: {e}")
 
     # Additional cleanup for any resources that might have been missed
-    for resource in PlatformResource.search_by_key(graph_client, "test_"):
+    for resource in PlatformResource.search_by_filters(
+        graph_client,
+        ElasticPlatformResourceQuery.create_from().add_wildcard(
+            PlatformResourceSearchFields.PRIMARY_KEY, "test_*"
+        ),
+    ):
         try:
             resource.delete(graph_client)
         except Exception as e:
@@ -99,3 +100,141 @@ def test_platform_resource_search(graph_client, test_id, cleanup_resources):
     ]
     assert len(search_results) == 1
     assert search_results[0] == platform_resource
+
+
+def test_platform_resource_non_existent(graph_client, test_id):
+    key = PlatformResourceKey(
+        platform=f"test_platform_{test_id}",
+        resource_type=f"test_resource_type_{test_id}",
+        primary_key=f"test_primary_key_{test_id}",
+    )
+    platform_resource = PlatformResource.from_datahub(
+        key=key,
+        graph_client=graph_client,
+    )
+    assert platform_resource is None
+
+
+def test_platform_resource_urn_secondary_key(graph_client, test_id, cleanup_resources):
+    key = PlatformResourceKey(
+        platform=f"test_platform_{test_id}",
+        resource_type=f"test_resource_type_{test_id}",
+        primary_key=f"test_primary_key_{test_id}",
+    )
+    dataset_urn = (
+        f"urn:li:dataset:(urn:li:dataPlatform:test,test_secondary_key_{test_id},PROD)"
+    )
+    platform_resource = PlatformResource.create(
+        key=key,
+        value={"test_key": f"test_value_{test_id}"},
+        secondary_keys=[dataset_urn],
+    )
+    platform_resource.to_datahub(graph_client)
+    cleanup_resources.append(platform_resource)
+    wait_for_writes_to_sync()
+
+    read_platform_resources = [
+        r
+        for r in PlatformResource.search_by_key(
+            graph_client, dataset_urn, primary=False
+        )
+    ]
+    assert len(read_platform_resources) == 1
+    assert read_platform_resources[0] == platform_resource
+
+
+def test_platform_resource_listing_by_resource_type(
+    graph_client, test_id, cleanup_resources
+):
+    # Generate two resources with the same resource type
+    key1 = PlatformResourceKey(
+        platform=f"test_platform_{test_id}",
+        resource_type=f"test_resource_type_{test_id}",
+        primary_key=f"test_primary_key_1_{test_id}",
+    )
+    platform_resource1 = PlatformResource.create(
+        key=key1,
+        value={"test_key": f"test_value_1_{test_id}"},
+    )
+    platform_resource1.to_datahub(graph_client)
+
+    key2 = PlatformResourceKey(
+        platform=f"test_platform_{test_id}",
+        resource_type=f"test_resource_type_{test_id}",
+        primary_key=f"test_primary_key_2_{test_id}",
+    )
+    platform_resource2 = PlatformResource.create(
+        key=key2,
+        value={"test_key": f"test_value_2_{test_id}"},
+    )
+    platform_resource2.to_datahub(graph_client)
+
+    wait_for_writes_to_sync()
+
+    search_results = [
+        r
+        for r in PlatformResource.search_by_filters(
+            graph_client,
+            query=ElasticPlatformResourceQuery.create_from(
+                (PlatformResourceSearchFields.RESOURCE_TYPE, key1.resource_type)
+            ),
+        )
+    ]
+    assert len(search_results) == 2
+
+    read_platform_resource_1 = next(r for r in search_results if r.id == key1.id)
+    read_platform_resource_2 = next(r for r in search_results if r.id == key2.id)
+    assert read_platform_resource_1 == platform_resource1
+    assert read_platform_resource_2 == platform_resource2
+
+
+def test_platform_resource_listing_complex_queries(graph_client, test_id):
+    # Generate two resources with the same resource type
+    key1 = PlatformResourceKey(
+        platform=f"test_platform1_{test_id}",
+        resource_type=f"test_resource_type_{test_id}",
+        primary_key=f"test_primary_key_1_{test_id}",
+    )
+    platform_resource1 = PlatformResource.create(
+        key=key1,
+        value={"test_key": f"test_value_1_{test_id}"},
+    )
+    platform_resource1.to_datahub(graph_client)
+
+    key2 = PlatformResourceKey(
+        platform=f"test_platform2_{test_id}",
+        resource_type=f"test_resource_type_{test_id}",
+        primary_key=f"test_primary_key_2_{test_id}",
+    )
+    platform_resource2 = PlatformResource.create(
+        key=key2,
+        value={"test_key": f"test_value_2_{test_id}"},
+    )
+    platform_resource2.to_datahub(graph_client)
+
+    wait_for_writes_to_sync()
+    from datahub.api.entities.platformresource.platform_resource import (
+        ElasticPlatformResourceQuery,
+        LogicalOperator,
+        PlatformResourceSearchFields,
+    )
+
+    query = (
+        ElasticPlatformResourceQuery.create_from()
+        .group(LogicalOperator.AND)
+        .add_field_match(PlatformResourceSearchFields.RESOURCE_TYPE, key1.resource_type)
+        .add_field_not_match(PlatformResourceSearchFields.PLATFORM, key1.platform)
+        .end()
+    )
+
+    search_results = [
+        r
+        for r in PlatformResource.search_by_filters(
+            graph_client,
+            query=query,
+        )
+    ]
+    assert len(search_results) == 1
+
+    read_platform_resource = search_results[0]
+    assert read_platform_resource == platform_resource2
diff --git a/smoke-test/tests/restli/__init__.py b/smoke-test/tests/restli/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/smoke-test/tests/restli/restli_test.py b/smoke-test/tests/restli/restli_test.py
new file mode 100644
index 0000000000000..7e9cf6aa76235
--- /dev/null
+++ b/smoke-test/tests/restli/restli_test.py
@@ -0,0 +1,98 @@
+import dataclasses
+import json
+import time
+from typing import List
+
+import pytest
+from datahub.emitter.aspect import JSON_CONTENT_TYPE
+from datahub.emitter.mce_builder import make_dashboard_urn
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.serialization_helper import pre_json_transform
+from datahub.metadata.schema_classes import (
+    AuditStampClass,
+    ChangeAuditStampsClass,
+    DashboardInfoClass,
+    GenericAspectClass,
+    MetadataChangeProposalClass,
+)
+from datahub.utilities.urns.urn import guess_entity_type
+
+from tests.utils import delete_urns
+
+generated_urns: List[str] = []
+
+
+@dataclasses.dataclass
+class MetadataChangeProposalInvalidWrapper(MetadataChangeProposalWrapper):
+    @staticmethod
+    def _make_generic_aspect(dict) -> GenericAspectClass:
+        serialized = json.dumps(pre_json_transform(dict))
+        return GenericAspectClass(
+            value=serialized.encode(),
+            contentType=JSON_CONTENT_TYPE,
+        )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def __post_init__(self) -> None:
+        if self.entityUrn:
+            self.entityType = guess_entity_type(self.entityUrn)
+
+    def make_mcp(self) -> MetadataChangeProposalClass:
+        serializedAspect = None
+        if self.aspect is not None:
+            serializedAspect = (
+                MetadataChangeProposalInvalidWrapper._make_generic_aspect(self.aspect)
+            )
+
+        mcp = self._make_mcp_without_aspects()
+        mcp.aspect = serializedAspect
+        return mcp
+
+
+@pytest.fixture(scope="module")
+def ingest_cleanup_data(auth_session, graph_client, request):
+    yield
+    delete_urns(graph_client, generated_urns)
+
+
+def test_gms_ignore_unknown_dashboard_info(graph_client):
+    dashboard_urn = make_dashboard_urn(platform="looker", name="test-ignore-unknown")
+    generated_urns.extend([dashboard_urn])
+
+    audit_stamp = pre_json_transform(
+        ChangeAuditStampsClass(
+            created=AuditStampClass(
+                time=int(time.time() * 1000),
+                actor="urn:li:corpuser:datahub",
+            )
+        ).to_obj()
+    )
+
+    invalid_dashboard_info = {
+        "title": "Ignore Unknown Title",
+        "description": "Ignore Unknown Description",
+        "lastModified": audit_stamp,
+        "notAValidField": "invalid field value",
+    }
+    mcpw = MetadataChangeProposalInvalidWrapper(
+        entityUrn=dashboard_urn,
+        aspectName="dashboardInfo",
+        aspect=invalid_dashboard_info,
+    )
+
+    mcp = mcpw.make_mcp()
+    assert "notAValidField" in str(mcp)
+    assert "invalid field value" in str(mcp)
+
+    graph_client.emit_mcp(mcpw, async_flag=False)
+
+    dashboard_info = graph_client.get_aspect(
+        entity_urn=dashboard_urn,
+        aspect_type=DashboardInfoClass,
+    )
+
+    assert dashboard_info
+    assert dashboard_info.title == invalid_dashboard_info["title"]
+    assert dashboard_info.description == invalid_dashboard_info["description"]
diff --git a/smoke-test/tests/structured_properties/bad_entity_type.yaml b/smoke-test/tests/structured_properties/bad_entity_type.yaml
new file mode 100644
index 0000000000000..7e919a5a38390
--- /dev/null
+++ b/smoke-test/tests/structured_properties/bad_entity_type.yaml
@@ -0,0 +1,6 @@
+- id: clusterTypeBad
+  type: STRING
+  display_name: Cluster's type
+  description: 'Test Cluster Type Property'
+  entity_types:
+    - urn:li:entityType:dataset # should fail because this is not a valid entity type
diff --git a/smoke-test/tests/structured_properties/test_structured_properties.py b/smoke-test/tests/structured_properties/test_structured_properties.py
index 8b6fead789b3f..533a03a55735a 100644
--- a/smoke-test/tests/structured_properties/test_structured_properties.py
+++ b/smoke-test/tests/structured_properties/test_structured_properties.py
@@ -165,13 +165,13 @@ def get_property_from_entity(
     return None
 
 
-def to_es_name(property_name=None, namespace=default_namespace, qualified_name=None):
+def to_es_filter_name(
+    property_name=None, namespace=default_namespace, qualified_name=None
+):
     if property_name:
-        namespace_field = namespace.replace(".", "_")
-        return f"structuredProperties.{namespace_field}_{property_name}"
+        return f"structuredProperties.{namespace}.{property_name}"
     else:
-        escaped_qualified_name = qualified_name.replace(".", "_")
-        return f"structuredProperties.{escaped_qualified_name}"
+        return f"structuredProperties.{qualified_name}"
 
 
 # @tenacity.retry(
@@ -371,6 +371,24 @@ def test_structured_property_schema_field(ingest_cleanup_data, graph_client):
             raise e
 
 
+def test_structured_properties_yaml_load_with_bad_entity_type(
+    ingest_cleanup_data, graph_client
+):
+    try:
+        StructuredProperties.create(
+            "tests/structured_properties/bad_entity_type.yaml",
+            graph=graph_client,
+        )
+        raise AssertionError(
+            "Should not be able to create structured properties with bad entity type"
+        )
+    except Exception as e:
+        if "urn:li:entityType:dataset is not a valid entity type urn" in str(e):
+            pass
+        else:
+            raise e
+
+
 def test_dataset_yaml_loader(ingest_cleanup_data, graph_client):
     StructuredProperties.create(
         "tests/structured_properties/test_structured_properties.yaml",
@@ -446,7 +464,7 @@ def test_structured_property_search(
         graph_client.get_urns_by_filter(
             extraFilters=[
                 {
-                    "field": to_es_name(dataset_property_name),
+                    "field": to_es_filter_name(dataset_property_name),
                     "negated": "false",
                     "condition": "EXISTS",
                 }
@@ -475,7 +493,7 @@ def test_structured_property_search(
             entity_types=["tag"],
             extraFilters=[
                 {
-                    "field": to_es_name(
+                    "field": to_es_filter_name(
                         field_property_name, namespace="io.datahubproject.test"
                     ),
                     "negated": "false",
@@ -492,7 +510,7 @@ def test_structured_property_search(
             entity_types=["dataset", "tag"],
             extraFilters=[
                 {
-                    "field": to_es_name(dataset_property_name),
+                    "field": to_es_filter_name(dataset_property_name),
                     "negated": "false",
                     "condition": "EXISTS",
                 }
@@ -690,7 +708,7 @@ def test_dataset_structured_property_soft_delete_search_filter_validation(
         graph_client.get_urns_by_filter(
             extraFilters=[
                 {
-                    "field": to_es_name(dataset_property_name),
+                    "field": to_es_filter_name(property_name=dataset_property_name),
                     "negated": "false",
                     "condition": "EXISTS",
                 }
@@ -710,7 +728,7 @@ def test_dataset_structured_property_soft_delete_search_filter_validation(
             graph_client.get_urns_by_filter(
                 extraFilters=[
                     {
-                        "field": to_es_name(dataset_property_name),
+                        "field": to_es_filter_name(property_name=dataset_property_name),
                         "negated": "false",
                         "condition": "EXISTS",
                     }
@@ -779,7 +797,7 @@ def validate_search(qualified_name, expected):
             graph_client.get_urns_by_filter(
                 extraFilters=[
                     {
-                        "field": to_es_name(qualified_name=qualified_name),
+                        "field": to_es_filter_name(qualified_name=qualified_name),
                         "negated": "false",
                         "condition": "EXISTS",
                     }
diff --git a/smoke-test/tests/structured_properties/test_structured_properties.yaml b/smoke-test/tests/structured_properties/test_structured_properties.yaml
index 569a3d185165d..eee8d53aeccf5 100644
--- a/smoke-test/tests/structured_properties/test_structured_properties.yaml
+++ b/smoke-test/tests/structured_properties/test_structured_properties.yaml
@@ -1,13 +1,13 @@
 - id: clusterType
   type: STRING
   display_name: Cluster's type
-  description: "Test Cluster Type Property"
+  description: 'Test Cluster Type Property'
   entity_types:
     - dataset
 - id: clusterName
   type: STRING
   display_name: Cluster's name
-  description: "Test Cluster Name Property"
+  description: 'Test Cluster Name Property'
   entity_types:
     - dataset
 - id: projectNames
@@ -15,9 +15,9 @@
   cardinality: MULTIPLE
   display_name: Project Name
   entity_types:
-    - dataset # or urn:li:logicalEntity:metamodel.datahub.dataset
-    - dataflow
-  description: "Test property for project name"
+    - dataset # or urn:li:entityType:datahub.dataset
+    - dataFlow
+  description: 'Test property for project name'
   allowed_values:
     - value: Tracking
       description: test value 1 for project
diff --git a/smoke-test/tests/test_stateful_ingestion.py b/smoke-test/tests/test_stateful_ingestion.py
index c0df51dd9d98e..4436cf26c2fd7 100644
--- a/smoke-test/tests/test_stateful_ingestion.py
+++ b/smoke-test/tests/test_stateful_ingestion.py
@@ -65,10 +65,6 @@ def get_current_checkpoint_from_pipeline(
             "enabled": True,
             "remove_stale_metadata": True,
             "fail_safe_threshold": 100.0,
-            "state_provider": {
-                "type": "datahub",
-                "config": {"datahub_api": {"server": auth_session.gms_url()}},
-            },
         },
     }
 
diff --git a/smoke-test/tests/tests/data.json b/smoke-test/tests/tests/data.json
index b91c9d9f0ec3b..7dccc70457e71 100644
--- a/smoke-test/tests/tests/data.json
+++ b/smoke-test/tests/tests/data.json
@@ -148,7 +148,7 @@
     "changeType": "UPSERT",
     "aspectName": "testInfo",
     "aspect": {
-      "value": "{\"name\": \"Sample Test 1\", \"category\": \"Examples\", \"description\": \"An example of Metadata Test\", \"definition\": { \"type\": \"JSON\", \"json\": \"{\\\"on\\\":{\\\"dataset\\\":[{\\\"query\\\":\\\"dataPlatformInstance.platform\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:dataPlatform:bigQuery\\\"]}]},\\\"rules\\\":{\\\"or\\\":[{\\\"query\\\":\\\"glossaryTerms.terms.glossaryTermInfo.parentNode\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:glossaryNode:Category\\\"]},{\\\"query\\\":\\\"container.container.glossaryTerms.terms.glossaryTermInfo.parentNode\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:glossaryNode:Category\\\"]}]}}\"} }",
+      "value": "{\"name\": \"Sample Test 1\", \"category\": \"Examples\", \"description\": \"An example of Metadata Test\", \"definition\": { \"type\": \"JSON\", \"json\": \"{\\\"on\\\":{\\\"types\\\":[\\\"dataset\\\"],\\\"dataset\\\":[{\\\"query\\\":\\\"dataPlatformInstance.platform\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:dataPlatform:bigQuery\\\"]}]},\\\"rules\\\":{\\\"or\\\":[{\\\"query\\\":\\\"glossaryTerms.terms.glossaryTermInfo.parentNode\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:glossaryNode:Category\\\"]},{\\\"query\\\":\\\"container.container.glossaryTerms.terms.glossaryTermInfo.parentNode\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:glossaryNode:Category\\\"]}]}}\"} }",
       "contentType": "application/json"
     },
     "systemMetadata": null
@@ -161,7 +161,7 @@
     "changeType": "UPSERT",
     "aspectName": "testInfo",
     "aspect": {
-      "value": "{\"name\": \"Sample Test 2\", \"category\": \"Examples\", \"description\": \"An example of another Metadata Test\", \"definition\": { \"type\": \"JSON\", \"json\": \"{\\\"on\\\":{\\\"dataset\\\":[{\\\"query\\\":\\\"dataPlatformInstance.platform\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:dataPlatform:bigQuery\\\"]}]},\\\"rules\\\":{\\\"or\\\":[{\\\"query\\\":\\\"glossaryTerms.terms.glossaryTermInfo.parentNode\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:glossaryNode:Category\\\"]},{\\\"query\\\":\\\"container.container.glossaryTerms.terms.glossaryTermInfo.parentNode\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:glossaryNode:Category\\\"]}]}}\"} }",
+      "value": "{\"name\": \"Sample Test 2\", \"category\": \"Examples\", \"description\": \"An example of another Metadata Test\", \"definition\": { \"type\": \"JSON\", \"json\": \"{\\\"on\\\":{\\\"types\\\":[\\\"dataset\\\"],\\\"dataset\\\":[{\\\"query\\\":\\\"dataPlatformInstance.platform\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:dataPlatform:bigQuery\\\"]}]},\\\"rules\\\":{\\\"or\\\":[{\\\"query\\\":\\\"glossaryTerms.terms.glossaryTermInfo.parentNode\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:glossaryNode:Category\\\"]},{\\\"query\\\":\\\"container.container.glossaryTerms.terms.glossaryTermInfo.parentNode\\\",\\\"condition\\\":\\\"EQUALS\\\",\\\"values\\\":[\\\"urn:li:glossaryNode:Category\\\"]}]}}\"} }",
       "contentType": "application/json"
     },
     "systemMetadata": null
diff --git a/smoke-test/tests/tests/tests_test.py b/smoke-test/tests/tests/tests_test.py
index bc9ebe46c5279..8e6af2f3f5bf4 100644
--- a/smoke-test/tests/tests/tests_test.py
+++ b/smoke-test/tests/tests/tests_test.py
@@ -1,10 +1,20 @@
+import time
+from typing import List
+
 import pytest
 import tenacity
 
-from tests.utils import delete_urns_from_file, get_sleep_info, ingest_file_via_rest
+from tests.utils import (
+    delete_urns,
+    delete_urns_from_file,
+    get_sleep_info,
+    ingest_file_via_rest,
+)
 
 sleep_sec, sleep_times = get_sleep_info()
 
+TEST_URNS: List[str] = []
+
 
 @pytest.fixture(scope="module", autouse=True)
 def ingest_cleanup_data(auth_session, graph_client, request):
@@ -12,17 +22,20 @@ def ingest_cleanup_data(auth_session, graph_client, request):
     ingest_file_via_rest(auth_session, "tests/tests/data.json")
     yield
     print("removing test data")
+    delete_urns(graph_client, TEST_URNS)
     delete_urns_from_file(graph_client, "tests/tests/data.json")
 
 
-test_id = "test id"
 test_name = "test name"
 test_category = "test category"
 test_description = "test description"
 test_description = "test description"
 
 
-def create_test(auth_session):
+def create_test(auth_session, test_id="test id"):
+    test_id = f"{test_id}_{int(time.time())}"
+    TEST_URNS.extend([f"urn:li:test:{test_id}"])
+
     # Create new Test
     create_test_json = {
         "query": """mutation createTest($input: CreateTestInput!) {\n
@@ -53,21 +66,50 @@ def create_test(auth_session):
     return res_data["data"]["createTest"]
 
 
-def delete_test(auth_session, test_urn):
-    delete_test_json = {
-        "query": """mutation deleteTest($urn: String!) {\n
-            deleteTest(urn: $urn)
+@pytest.mark.dependency()
+def test_get_test_results(auth_session):
+    urn = (
+        "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tests-sample,PROD)"  # Test urn
+    )
+    json = {
+        "query": """query getDataset($urn: String!) {\n
+            dataset(urn: $urn) {\n
+                urn\n
+                testResults {\n
+                    failing {\n
+                      test {\n
+                        urn\n
+                      }\n
+                      type
+                    }\n
+                    passing {\n
+                      test {\n
+                        urn\n
+                      }\n
+                      type
+                    }\n
+                }\n
+            }\n
         }""",
-        "variables": {"urn": test_urn},
+        "variables": {"urn": urn},
     }
-
     response = auth_session.post(
-        f"{auth_session.frontend_url()}/api/v2/graphql", json=delete_test_json
+        f"{auth_session.frontend_url()}/api/v2/graphql", json=json
     )
     response.raise_for_status()
+    res_data = response.json()
 
+    assert res_data
+    assert res_data["data"]
+    assert res_data["data"]["dataset"]
+    assert res_data["data"]["dataset"]["urn"] == urn
+    assert res_data["data"]["dataset"]["testResults"] == {
+        "failing": [{"test": {"urn": "urn:li:test:test-1"}, "type": "FAILURE"}],
+        "passing": [{"test": {"urn": "urn:li:test:test-2"}, "type": "SUCCESS"}],
+    }
 
-@pytest.mark.dependency()
+
+@pytest.mark.dependency(depends=["test_get_test_results"])
 def test_create_test(auth_session):
     test_urn = create_test(auth_session)
 
@@ -105,17 +147,14 @@ def test_create_test(auth_session):
     }
     assert "errors" not in res_data
 
-    # Delete test
-    delete_test(auth_session, test_urn)
-
-    # Ensure the test no longer exists
+    # Ensure that soft-deleted tests
     response = auth_session.post(
         f"{auth_session.frontend_url()}/api/v2/graphql", json=get_test_json
     )
     response.raise_for_status()
     res_data = response.json()
 
-    assert res_data["data"]["test"] is None
+    assert res_data["data"]["test"] is not None
     assert "errors" not in res_data
 
 
@@ -125,7 +164,6 @@ def test_update_test(auth_session):
     test_name = "new name"
     test_category = "new category"
     test_description = "new description"
-    test_description = "new description"
 
     # Update Test
     update_test_json = {
@@ -188,8 +226,6 @@ def test_update_test(auth_session):
     }
     assert "errors" not in res_data
 
-    delete_test(auth_session, test_urn)
-
 
 @tenacity.retry(
     stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec)
@@ -225,45 +261,3 @@ def test_list_tests_retries(auth_session):
 @pytest.mark.dependency(depends=["test_update_test"])
 def test_list_tests(auth_session):
     test_list_tests_retries(auth_session)
-
-
-def test_get_test_results(auth_session):
-    urn = (
-        "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tests-sample,PROD)"  # Test urn
-    )
-    json = {
-        "query": """query getDataset($urn: String!) {\n
-            dataset(urn: $urn) {\n
-                urn\n
-                testResults {\n
-                    failing {\n
-                      test {\n
-                        urn\n
-                      }\n
-                      type
-                    }\n
-                    passing {\n
-                      test {\n
-                        urn\n
-                      }\n
-                      type
-                    }\n
-                }\n
-            }\n
-        }""",
-        "variables": {"urn": urn},
-    }
-    response = auth_session.post(
-        f"{auth_session.frontend_url()}/api/v2/graphql", json=json
-    )
-    response.raise_for_status()
-    res_data = response.json()
-
-    assert res_data
-    assert res_data["data"]
-    assert res_data["data"]["dataset"]
-    assert res_data["data"]["dataset"]["urn"] == urn
-    assert res_data["data"]["dataset"]["testResults"] == {
-        "failing": [{"test": {"urn": "urn:li:test:test-1"}, "type": "FAILURE"}],
-        "passing": [{"test": {"urn": "urn:li:test:test-2"}, "type": "SUCCESS"}],
-    }
diff --git a/smoke-test/tests/tokens/revokable_access_token_test.py b/smoke-test/tests/tokens/revokable_access_token_test.py
index 0ae18c32662c5..006daae39333e 100644
--- a/smoke-test/tests/tokens/revokable_access_token_test.py
+++ b/smoke-test/tests/tokens/revokable_access_token_test.py
@@ -9,6 +9,8 @@
     wait_for_writes_to_sync,
 )
 
+from .token_utils import listUsers, removeUser
+
 pytestmark = pytest.mark.no_cypress_suite1
 
 # Disable telemetry
@@ -17,11 +19,6 @@
 (admin_user, admin_pass) = get_admin_credentials()
 
 
-@pytest.fixture(autouse=True)
-def setup(auth_session):
-    wait_for_writes_to_sync()
-
-
 @pytest.fixture()
 def auth_exclude_filter():
     return {
@@ -495,45 +492,3 @@ def getAccessTokenMetadata(session, token):
     response.raise_for_status()
 
     return response.json()
-
-
-def removeUser(session, urn):
-    # Remove user
-    json = {
-        "query": """mutation removeUser($urn: String!) {
-            removeUser(urn: $urn)
-        }""",
-        "variables": {"urn": urn},
-    }
-
-    response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
-
-    response.raise_for_status()
-    return response.json()
-
-
-def listUsers(session):
-    input = {
-        "start": "0",
-        "count": "20",
-    }
-
-    # list users
-    json = {
-        "query": """query listUsers($input: ListUsersInput!) {
-            listUsers(input: $input) {
-              start
-              count
-              total
-              users {
-                username
-              }
-            }
-        }""",
-        "variables": {"input": input},
-    }
-
-    response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
-
-    response.raise_for_status()
-    return response.json()
diff --git a/smoke-test/tests/tokens/session_access_token_test.py b/smoke-test/tests/tokens/session_access_token_test.py
new file mode 100644
index 0000000000000..a16abc4445303
--- /dev/null
+++ b/smoke-test/tests/tokens/session_access_token_test.py
@@ -0,0 +1,173 @@
+import os
+import time
+
+import pytest
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.schema_classes import AuditStampClass, CorpUserStatusClass
+from requests.exceptions import HTTPError
+
+from tests.utils import (
+    get_admin_credentials,
+    get_frontend_url,
+    login_as,
+    wait_for_writes_to_sync,
+)
+
+from .token_utils import getUserId, listUsers, removeUser
+
+pytestmark = pytest.mark.no_cypress_suite1
+
+# Disable telemetry
+os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false"
+
+(admin_user, admin_pass) = get_admin_credentials()
+user_urn = "urn:li:corpuser:sessionUser"
+
+
+@pytest.fixture(scope="class")
+def custom_user_session():
+    """Fixture to execute setup before and tear down after all tests are run"""
+    admin_session = login_as(admin_user, admin_pass)
+
+    res_data = removeUser(admin_session, user_urn)
+    assert res_data
+    assert "error" not in res_data
+
+    # Test getting the invite token
+    get_invite_token_json = {
+        "query": """query getInviteToken($input: GetInviteTokenInput!) {
+            getInviteToken(input: $input){
+              inviteToken
+            }
+        }""",
+        "variables": {"input": {}},
+    }
+
+    get_invite_token_response = admin_session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=get_invite_token_json
+    )
+    get_invite_token_response.raise_for_status()
+    get_invite_token_res_data = get_invite_token_response.json()
+
+    assert get_invite_token_res_data
+    assert get_invite_token_res_data["data"]
+    invite_token = get_invite_token_res_data["data"]["getInviteToken"]["inviteToken"]
+    assert invite_token is not None
+    assert "error" not in invite_token
+
+    # Pass the invite token when creating the user
+    sign_up_json = {
+        "fullName": "Test Session User",
+        "email": "sessionUser",
+        "password": "sessionUser",
+        "title": "Date Engineer",
+        "inviteToken": invite_token,
+    }
+
+    sign_up_response = admin_session.post(
+        f"{get_frontend_url()}/signUp", json=sign_up_json
+    )
+    sign_up_response.raise_for_status()
+    assert sign_up_response
+    assert "error" not in sign_up_response
+    # Sleep for eventual consistency
+    wait_for_writes_to_sync()
+
+    # signUp will override the session cookie to the new user to be signed up.
+    admin_session.cookies.clear()
+    admin_session = login_as(admin_user, admin_pass)
+
+    # Make user created user is there.
+    res_data = listUsers(admin_session)
+    assert res_data["data"]
+    assert res_data["data"]["listUsers"]
+    assert {"username": "sessionUser"} in res_data["data"]["listUsers"]["users"]
+
+    yield login_as(sign_up_json["email"], sign_up_json["password"])
+
+    # Delete created user
+    res_data = removeUser(admin_session, user_urn)
+    assert res_data
+    assert res_data["data"]
+    assert res_data["data"]["removeUser"] is True
+    # Sleep for eventual consistency
+    wait_for_writes_to_sync()
+
+    # Make user created user is not there.
+    res_data = listUsers(admin_session)
+    assert res_data["data"]
+    assert res_data["data"]["listUsers"]
+    assert {"username": "sessionUser"} not in res_data["data"]["listUsers"]["users"]
+
+
+@pytest.mark.dependency()
+def test_soft_delete(graph_client, custom_user_session):
+    # assert initial access
+    assert getUserId(custom_user_session) == {"urn": user_urn}
+
+    graph_client.soft_delete_entity(urn=user_urn)
+    wait_for_writes_to_sync()
+
+    with pytest.raises(HTTPError) as req_info:
+        getUserId(custom_user_session)
+    assert "403 Client Error: Forbidden" in str(req_info.value)
+
+    # undo soft delete
+    graph_client.set_soft_delete_status(urn=user_urn, delete=False)
+    wait_for_writes_to_sync()
+
+
+@pytest.mark.dependency(depends=["test_soft_delete"])
+def test_suspend(graph_client, custom_user_session):
+    # assert initial access
+    assert getUserId(custom_user_session) == {"urn": user_urn}
+
+    graph_client.emit(
+        MetadataChangeProposalWrapper(
+            entityType="corpuser",
+            entityUrn=user_urn,
+            changeType="UPSERT",
+            aspectName="corpUserStatus",
+            aspect=CorpUserStatusClass(
+                status="SUSPENDED",
+                lastModified=AuditStampClass(
+                    time=int(time.time() * 1000.0), actor="urn:li:corpuser:unknown"
+                ),
+            ),
+        )
+    )
+    wait_for_writes_to_sync()
+
+    with pytest.raises(HTTPError) as req_info:
+        getUserId(custom_user_session)
+    assert "403 Client Error: Forbidden" in str(req_info.value)
+
+    # undo suspend
+    graph_client.emit(
+        MetadataChangeProposalWrapper(
+            entityType="corpuser",
+            entityUrn=user_urn,
+            changeType="UPSERT",
+            aspectName="corpUserStatus",
+            aspect=CorpUserStatusClass(
+                status="ACTIVE",
+                lastModified=AuditStampClass(
+                    time=int(time.time() * 1000.0), actor="urn:li:corpuser:unknown"
+                ),
+            ),
+        )
+    )
+    wait_for_writes_to_sync()
+
+
+@pytest.mark.dependency(depends=["test_suspend"])
+def test_hard_delete(graph_client, custom_user_session):
+    # assert initial access
+    assert getUserId(custom_user_session) == {"urn": user_urn}
+
+    graph_client.hard_delete_entity(urn=user_urn)
+    wait_for_writes_to_sync()
+
+    with pytest.raises(HTTPError) as req_info:
+        getUserId(custom_user_session)
+    assert "403 Client Error: Forbidden" in str(req_info.value)
diff --git a/smoke-test/tests/tokens/token_utils.py b/smoke-test/tests/tokens/token_utils.py
new file mode 100644
index 0000000000000..10558e7085de7
--- /dev/null
+++ b/smoke-test/tests/tokens/token_utils.py
@@ -0,0 +1,53 @@
+from tests.utils import get_frontend_url
+
+
+def getUserId(session):
+    response = session.get(
+        f"{get_frontend_url()}/openapi/operations/identity/user/urn",
+        params={"skipCache": "true"},
+    )
+
+    response.raise_for_status()
+    return response.json()
+
+
+def removeUser(session, urn):
+    # Remove user
+    json = {
+        "query": """mutation removeUser($urn: String!) {
+            removeUser(urn: $urn)
+        }""",
+        "variables": {"urn": urn},
+    }
+
+    response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
+
+    response.raise_for_status()
+    return response.json()
+
+
+def listUsers(session):
+    input = {
+        "start": "0",
+        "count": "20",
+    }
+
+    # list users
+    json = {
+        "query": """query listUsers($input: ListUsersInput!) {
+            listUsers(input: $input) {
+              start
+              count
+              total
+              users {
+                username
+              }
+            }
+        }""",
+        "variables": {"input": input},
+    }
+
+    response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
+
+    response.raise_for_status()
+    return response.json()
diff --git a/smoke-test/tests/utilities/concurrent_openapi.py b/smoke-test/tests/utilities/concurrent_openapi.py
new file mode 100644
index 0000000000000..076cd12c7a3b3
--- /dev/null
+++ b/smoke-test/tests/utilities/concurrent_openapi.py
@@ -0,0 +1,116 @@
+import concurrent.futures
+import glob
+import json
+import logging
+import time
+
+from deepdiff import DeepDiff
+
+logger = logging.getLogger(__name__)
+
+
+def load_tests(fixture_glob):
+    """
+    Scans a directory structure looking for json files which define expected tests/responses
+    :param fixture_glob: Glob path such as "tests/openapi/**/*.json"
+    :return: tuples of the filename and dictionary of the file content
+    """
+    for test_fixture in glob.glob(fixture_glob):
+        with open(test_fixture) as f:
+            yield (test_fixture, json.load(f))
+
+
+def execute_request(auth_session, request):
+    """
+    Based on the request dictionary execute the request against gms
+    :param auth_session: authentication
+    :param request: request dictionary
+    :return: output of the request
+    """
+    if "method" in request:
+        method = request.pop("method")
+    else:
+        method = "post"
+
+    url = auth_session.gms_url() + request.pop("url")
+
+    return getattr(auth_session, method)(url, **request)
+
+
+def evaluate_test(auth_session, test_name, test_data):
+    """
+    For each test step, execute the request and assert the expected response
+    :param auth_session: authentication
+    :param test_name: name of the test
+    :param test_data: test steps as defined in the test file
+    :return: none
+    """
+    try:
+        assert isinstance(test_data, list), "Expected test_data is a list of test steps"
+        for idx, req_resp in enumerate(test_data):
+            if "description" in req_resp["request"]:
+                description = req_resp["request"].pop("description")
+            else:
+                description = None
+            if "wait" in req_resp["request"]:
+                time.sleep(req_resp["request"]["wait"])
+                continue
+            url = req_resp["request"]["url"]
+            actual_resp = execute_request(auth_session, req_resp["request"])
+            try:
+                if "response" in req_resp and "status_codes" in req_resp["response"]:
+                    assert (
+                        actual_resp.status_code in req_resp["response"]["status_codes"]
+                    )
+                else:
+                    assert actual_resp.status_code in [200, 202, 204]
+                if "response" in req_resp:
+                    if "json" in req_resp["response"]:
+                        if "exclude_regex_paths" in req_resp["response"]:
+                            exclude_regex_paths = req_resp["response"][
+                                "exclude_regex_paths"
+                            ]
+                        else:
+                            exclude_regex_paths = []
+                        diff = DeepDiff(
+                            actual_resp.json(),
+                            req_resp["response"]["json"],
+                            exclude_regex_paths=exclude_regex_paths,
+                            ignore_order=True,
+                        )
+                        assert not diff
+                    else:
+                        logger.warning("No expected response json found")
+            except Exception as e:
+                logger.error(
+                    f"Error executing step: {idx}, url: {url}, test: {test_name}"
+                )
+                if description:
+                    logger.error(f"Step {idx} Description: {description}")
+                logger.error(f"Response content: {actual_resp.content}")
+                raise e
+    except Exception as e:
+        logger.error(f"Error executing test: {test_name}")
+        raise e
+
+
+def run_tests(auth_session, fixture_globs, num_workers=3):
+    """
+    Given a collection of test files, run them in parallel using N workers
+    :param auth_session: authentication
+    :param fixture_globs: test files
+    :param num_workers: concurrency
+    :return: none
+    """
+    with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
+        futures = []
+        for fixture_glob in fixture_globs:
+            for test_fixture, test_data in load_tests(fixture_glob=fixture_glob):
+                futures.append(
+                    executor.submit(
+                        evaluate_test, auth_session, test_fixture, test_data
+                    )
+                )
+
+        for future in concurrent.futures.as_completed(futures):
+            logger.info(future.result())
diff --git a/test-models/src/main/pegasus/com/datahub/test/testing/RelationshipBar.pdl b/test-models/src/main/pegasus/com/datahub/test/testing/RelationshipBar.pdl
index fca2ebb81f2f8..78083be40fdd3 100644
--- a/test-models/src/main/pegasus/com/datahub/test/testing/RelationshipBar.pdl
+++ b/test-models/src/main/pegasus/com/datahub/test/testing/RelationshipBar.pdl
@@ -5,10 +5,9 @@ import com.linkedin.common.Urn
 /**
  * For unit tests
  */
-@pairings = [ {
-  "destination" : "com.linkedin.common.urn.Urn",
-  "source" : "com.linkedin.common.urn.Urn"
-} ]
+@Aspect = {
+  "name": "relationshipBar"
+}
 record RelationshipBar {
 
   /**
@@ -19,5 +18,8 @@ record RelationshipBar {
   /**
    * For unit tests
    */
+  @Relationship = {
+    "name": "RelationshipBar"
+  }
   destination: Urn
 }
\ No newline at end of file
diff --git a/test-models/src/main/pegasus/com/datahub/test/testing/RelationshipFoo.pdl b/test-models/src/main/pegasus/com/datahub/test/testing/RelationshipFoo.pdl
index b9698ab8e4c51..5ce7f11f045c9 100644
--- a/test-models/src/main/pegasus/com/datahub/test/testing/RelationshipFoo.pdl
+++ b/test-models/src/main/pegasus/com/datahub/test/testing/RelationshipFoo.pdl
@@ -5,10 +5,6 @@ import com.linkedin.common.Urn
 /**
  * For unit tests
  */
-@pairings = [ {
-  "destination" : "com.linkedin.common.urn.Urn",
-  "source" : "com.linkedin.common.urn.Urn"
-} ]
 record RelationshipFoo {
 
   /**